Files
SCrawler/SCrawler/Download/Automation/Scheduler.vb
Andy b2a9b22478 2023.4.28.0
Plugins
IPluginContentProvider: added DownloadSingleObject function; added tokens to GetMedia and Download functions; removed GetSpecialData function
Add IDownloadableMedia interface
Removed 'Channel' option from all functions and enums
ISiteSettings: added GetSingleMediaInstance function
ExchangeOptions: removed 'IsChannel'
UserMediaTypes: added Audio and AudioPre enums
IUserMedia, PluginUserMedia: changed ContentType and DownloadState from integers to their enums

SCrawler
Add YouTube standalone downloader
Add gallery-dl & yt-dlp support
Remove 'UserInfo' requirement from 'ProfilesSaved'
Update 'SiteSettingsBase' to use domains and Netscape cookies
UserDataBase: remove channels; remove old 'Merge' const; standardize SavedPosts file naming; move 'ValidateMD5' function from Twitter to UserDataBase to use it in other UserData classes; add 'DownloadSingleObject' environment for single posts; add validating file extension for m3u8 during download; add reindex of video file during download

Rewritten DomainsContainer
Create a universal settings form and PSettingsArttribute
Gfycat, Imgur: turn these classes into IUserData to download a single object

All plugins: update 'GetInstance' function for saved posts; update domains where implemented; remove 'OptionForm' where it exists; update options where they exist; update unix date providers; reconfigure channels where they exist

LPSG: fix attachments; update converters and regex
Add sites: ThisVid, Mastodon, Pinterest, YouTube, YouTube music
Reddit: standardize container parsing for all data types; new channel environment; fix 'ReparseMissing' function; redirect data downloading to the base download function, saved crossposts support
Twitter: fixed gif path bug; fixed downloading saved posts
PornHub: hide unnecessary errors; photo galleries bug
RedGifs: add 'UserAgent' option

Added icons to download progress

Rename some objects
Completely redesigned standalone downloader form and rewritten its environment
WebClient2: update to use tokens

Labels: update label form (save labels to file only when OK button is clicked); change removing labels.txt from recycle bin to permanent; disable storing label 'NoParsedUser'

UserCreatorForm: remove the 'Channel' checkbox and related functions; ability to extract the user's URL from the buffer and apply parameters if found
Remove temporary 'EncryptCookies' module

MainFrame: added simplified way to create new users (Ctrl+Insert to create a new user with default parameters from clipboard URL); removed SCrawler command line argument "-v" (remove the ability to run SCrawler as video downloader)
PropertyValueHost: update for option forms compatibility
SettingsHost: removed 'GetSpecialData' fork; added 'GetSingleMediaInstance' fork
UserDataHost: update functions with tokens; update events; add 'DownloadSingleObject' function
Settings: add the ability to get environment from 4 destinations; add the ability to set the program environment manually; add CMDEncoding; add cache; remove the old function 'RemoveUnusedPlugins'; add 'STDownloader' properties; add YT compatibility; add new notification options; add deleting user settings file when 'SettingsCLS.Dispose()' if where are no users in SCrawler
UserFinder: remove old 'Merge' const; remove channel option
UserInfo: remove channel option
2023-04-28 10:13:46 +03:00

163 lines
7.4 KiB
VB.net

' Copyright (C) 2023 Andy https://github.com/AAndyProgram
' This program is free software: you can redistribute it and/or modify
' it under the terms of the GNU General Public License as published by
' the Free Software Foundation, either version 3 of the License, or
' (at your option) any later version.
'
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports System.Threading
Imports PersonalUtilities.Tools
Imports PersonalUtilities.Functions.XML
Imports SCrawler.DownloadObjects.Groups
Imports PauseModes = SCrawler.DownloadObjects.AutoDownloader.PauseModes
Namespace DownloadObjects
Friend Class Scheduler : Implements IEnumerable(Of AutoDownloader), IMyEnumerator(Of AutoDownloader), IDisposable
Friend Const Name_Plan As String = "Plan"
Friend Event PauseDisabled As AutoDownloader.PauseDisabledEventHandler
Private Sub OnPauseDisabled()
RaiseEvent PauseDisabled()
End Sub
Private ReadOnly Plans As List(Of AutoDownloader)
Private ReadOnly File As SFile = $"Settings\AutoDownload.xml"
Private ReadOnly PlanWorking As Predicate(Of AutoDownloader) = Function(Plan) Plan.Working
Private ReadOnly PlanDownloading As Predicate(Of AutoDownloader) = Function(Plan) Plan.Downloading
Private ReadOnly PlansWaiter As Action(Of Predicate(Of AutoDownloader)) = Sub(ByVal Predicate As Predicate(Of AutoDownloader))
While Plans.Exists(Predicate) : Thread.Sleep(200) : End While
End Sub
Friend Sub New()
Plans = New List(Of AutoDownloader)
If File.Exists Then
Using x As New XmlFile(File,, False) With {.AllowSameNames = True}
x.LoadData()
If x.Contains(Name_Plan) Then
For Each e In x : Plans.Add(New AutoDownloader(e)) : Next
Else
Plans.Add(New AutoDownloader(x))
End If
End Using
End If
If Plans.Count > 0 Then Plans.ForEach(Sub(p)
p.Source = Me
AddHandler p.PauseDisabled, AddressOf OnPauseDisabled
End Sub) : Plans.ListReindex
End Sub
Default Friend ReadOnly Property Item(ByVal Index As Integer) As AutoDownloader Implements IMyEnumerator(Of AutoDownloader).MyEnumeratorObject
Get
Return Plans(Index)
End Get
End Property
Friend ReadOnly Property Count As Integer Implements IMyEnumerator(Of AutoDownloader).MyEnumeratorCount
Get
Return Plans.Count
End Get
End Property
Friend Function NotificationClicked(ByVal Key As String, ByRef Found As Boolean, ByRef ActivateForm As Boolean) As Boolean
If Count > 0 Then
For Each plan As AutoDownloader In Plans
If plan.NotificationClicked(Key, Found, ActivateForm) Then Return True
Next
End If
Return False
End Function
Friend Sub Add(ByVal Plan As AutoDownloader)
Plan.Source = Me
AddHandler Plan.PauseDisabled, AddressOf OnPauseDisabled
Plans.Add(Plan)
Plans.ListReindex
Update()
End Sub
Friend Async Function RemoveAt(ByVal Index As Integer) As Task
If Index.ValueBetween(0, Count - 1) Then
With Plans(Index)
.Stop()
If .Working Then
Await Task.Run(Sub()
While .Working : Thread.Sleep(510) : End While
End Sub)
End If
.Dispose()
End With
Plans.RemoveAt(Index)
Plans.ListReindex
Update()
End If
End Function
Private _UpdateRequired As Boolean = False
Friend Sub Update()
_UpdateRequired = True
Try
If Plans.Count > 0 Then
Using x As New XmlFile With {.Name = "Scheduler", .AllowSameNames = True} : x.AddRange(Plans) : x.Save(File) : End Using
Else
File.Delete()
End If
_UpdateRequired = False
Catch
End Try
End Sub
#Region "Groups Support"
Friend Sub GROUPS_Updated(ByVal Sender As DownloadGroup)
If Count > 0 Then Plans.ForEach(Sub(p) p.GROUPS_Updated(Sender))
End Sub
Friend Sub GROUPS_Deleted(ByVal Sender As DownloadGroup)
If Count > 0 Then Plans.ForEach(Sub(p) p.GROUPS_Deleted(Sender))
End Sub
#End Region
#Region "Execution"
Friend Async Function Start(ByVal Init As Boolean) As Task
Await Task.Run(Sub()
If Count > 0 Then
If Plans.Exists(PlanDownloading) Then PlansWaiter(PlanDownloading)
For Each Plan In Plans
Plan.Start(Init)
PlansWaiter(PlanDownloading)
Thread.Sleep(1000)
Next
End If
End Sub)
End Function
Friend Sub [Stop]()
If Count > 0 Then Plans.ForEach(Sub(p) p.Stop())
End Sub
Friend Property Pause(Optional ByVal LimitDate As Date? = Nothing) As PauseModes
Get
If Count > 0 Then Return Plans.FirstOrDefault(Function(p) p.Pause >= PauseModes.Disabled).Pause Else Return PauseModes.Disabled
End Get
Set(ByVal p As PauseModes)
If Count > 0 Then Plans.ForEach(Sub(pp) pp.Pause(LimitDate) = p)
End Set
End Property
#End Region
#Region "IEnumerable Support"
Private Function GetEnumerator() As IEnumerator(Of AutoDownloader) Implements IEnumerable(Of AutoDownloader).GetEnumerator
Return New MyEnumerator(Of AutoDownloader)(Me)
End Function
Private Function IEnumerable_GetEnumerator() As IEnumerator Implements IEnumerable.GetEnumerator
Return GetEnumerator()
End Function
#End Region
#Region "IDisposable Support"
Private disposedValue As Boolean = False
Protected Overridable Overloads Sub Dispose(ByVal disposing As Boolean)
If Not disposedValue Then
If disposing Then
[Stop]()
If Plans.Exists(PlanWorking) Then Task.WaitAll(Task.Run(Sub() PlansWaiter(PlanWorking)))
If _UpdateRequired Then Update()
Plans.ListClearDispose
End If
disposedValue = True
End If
End Sub
Protected Overrides Sub Finalize()
Dispose(False)
MyBase.Finalize()
End Sub
Friend Overloads Sub Dispose() Implements IDisposable.Dispose
Dispose(True)
GC.SuppressFinalize(Me)
End Sub
#End Region
End Class
End Namespace