mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-15 00:02:17 +00:00
Plugins IPluginContentProvider: added DownloadSingleObject function; added tokens to GetMedia and Download functions; removed GetSpecialData function Add IDownloadableMedia interface Removed 'Channel' option from all functions and enums ISiteSettings: added GetSingleMediaInstance function ExchangeOptions: removed 'IsChannel' UserMediaTypes: added Audio and AudioPre enums IUserMedia, PluginUserMedia: changed ContentType and DownloadState from integers to their enums SCrawler Add YouTube standalone downloader Add gallery-dl & yt-dlp support Remove 'UserInfo' requirement from 'ProfilesSaved' Update 'SiteSettingsBase' to use domains and Netscape cookies UserDataBase: remove channels; remove old 'Merge' const; standardize SavedPosts file naming; move 'ValidateMD5' function from Twitter to UserDataBase to use it in other UserData classes; add 'DownloadSingleObject' environment for single posts; add validating file extension for m3u8 during download; add reindex of video file during download Rewritten DomainsContainer Create a universal settings form and PSettingsArttribute Gfycat, Imgur: turn these classes into IUserData to download a single object All plugins: update 'GetInstance' function for saved posts; update domains where implemented; remove 'OptionForm' where it exists; update options where they exist; update unix date providers; reconfigure channels where they exist LPSG: fix attachments; update converters and regex Add sites: ThisVid, Mastodon, Pinterest, YouTube, YouTube music Reddit: standardize container parsing for all data types; new channel environment; fix 'ReparseMissing' function; redirect data downloading to the base download function, saved crossposts support Twitter: fixed gif path bug; fixed downloading saved posts PornHub: hide unnecessary errors; photo galleries bug RedGifs: add 'UserAgent' option Added icons to download progress Rename some objects Completely redesigned standalone downloader form and rewritten its environment WebClient2: update to use tokens Labels: update label form (save labels to file only when OK button is clicked); change removing labels.txt from recycle bin to permanent; disable storing label 'NoParsedUser' UserCreatorForm: remove the 'Channel' checkbox and related functions; ability to extract the user's URL from the buffer and apply parameters if found Remove temporary 'EncryptCookies' module MainFrame: added simplified way to create new users (Ctrl+Insert to create a new user with default parameters from clipboard URL); removed SCrawler command line argument "-v" (remove the ability to run SCrawler as video downloader) PropertyValueHost: update for option forms compatibility SettingsHost: removed 'GetSpecialData' fork; added 'GetSingleMediaInstance' fork UserDataHost: update functions with tokens; update events; add 'DownloadSingleObject' function Settings: add the ability to get environment from 4 destinations; add the ability to set the program environment manually; add CMDEncoding; add cache; remove the old function 'RemoveUnusedPlugins'; add 'STDownloader' properties; add YT compatibility; add new notification options; add deleting user settings file when 'SettingsCLS.Dispose()' if where are no users in SCrawler UserFinder: remove old 'Merge' const; remove channel option UserInfo: remove channel option
163 lines
7.4 KiB
VB.net
163 lines
7.4 KiB
VB.net
' Copyright (C) 2023 Andy https://github.com/AAndyProgram
|
|
' This program is free software: you can redistribute it and/or modify
|
|
' it under the terms of the GNU General Public License as published by
|
|
' the Free Software Foundation, either version 3 of the License, or
|
|
' (at your option) any later version.
|
|
'
|
|
' This program is distributed in the hope that it will be useful,
|
|
' but WITHOUT ANY WARRANTY
|
|
Imports System.Threading
|
|
Imports PersonalUtilities.Tools
|
|
Imports PersonalUtilities.Functions.XML
|
|
Imports SCrawler.DownloadObjects.Groups
|
|
Imports PauseModes = SCrawler.DownloadObjects.AutoDownloader.PauseModes
|
|
Namespace DownloadObjects
|
|
Friend Class Scheduler : Implements IEnumerable(Of AutoDownloader), IMyEnumerator(Of AutoDownloader), IDisposable
|
|
Friend Const Name_Plan As String = "Plan"
|
|
Friend Event PauseDisabled As AutoDownloader.PauseDisabledEventHandler
|
|
Private Sub OnPauseDisabled()
|
|
RaiseEvent PauseDisabled()
|
|
End Sub
|
|
Private ReadOnly Plans As List(Of AutoDownloader)
|
|
Private ReadOnly File As SFile = $"Settings\AutoDownload.xml"
|
|
Private ReadOnly PlanWorking As Predicate(Of AutoDownloader) = Function(Plan) Plan.Working
|
|
Private ReadOnly PlanDownloading As Predicate(Of AutoDownloader) = Function(Plan) Plan.Downloading
|
|
Private ReadOnly PlansWaiter As Action(Of Predicate(Of AutoDownloader)) = Sub(ByVal Predicate As Predicate(Of AutoDownloader))
|
|
While Plans.Exists(Predicate) : Thread.Sleep(200) : End While
|
|
End Sub
|
|
Friend Sub New()
|
|
Plans = New List(Of AutoDownloader)
|
|
If File.Exists Then
|
|
Using x As New XmlFile(File,, False) With {.AllowSameNames = True}
|
|
x.LoadData()
|
|
If x.Contains(Name_Plan) Then
|
|
For Each e In x : Plans.Add(New AutoDownloader(e)) : Next
|
|
Else
|
|
Plans.Add(New AutoDownloader(x))
|
|
End If
|
|
End Using
|
|
End If
|
|
If Plans.Count > 0 Then Plans.ForEach(Sub(p)
|
|
p.Source = Me
|
|
AddHandler p.PauseDisabled, AddressOf OnPauseDisabled
|
|
End Sub) : Plans.ListReindex
|
|
End Sub
|
|
Default Friend ReadOnly Property Item(ByVal Index As Integer) As AutoDownloader Implements IMyEnumerator(Of AutoDownloader).MyEnumeratorObject
|
|
Get
|
|
Return Plans(Index)
|
|
End Get
|
|
End Property
|
|
Friend ReadOnly Property Count As Integer Implements IMyEnumerator(Of AutoDownloader).MyEnumeratorCount
|
|
Get
|
|
Return Plans.Count
|
|
End Get
|
|
End Property
|
|
Friend Function NotificationClicked(ByVal Key As String, ByRef Found As Boolean, ByRef ActivateForm As Boolean) As Boolean
|
|
If Count > 0 Then
|
|
For Each plan As AutoDownloader In Plans
|
|
If plan.NotificationClicked(Key, Found, ActivateForm) Then Return True
|
|
Next
|
|
End If
|
|
Return False
|
|
End Function
|
|
Friend Sub Add(ByVal Plan As AutoDownloader)
|
|
Plan.Source = Me
|
|
AddHandler Plan.PauseDisabled, AddressOf OnPauseDisabled
|
|
Plans.Add(Plan)
|
|
Plans.ListReindex
|
|
Update()
|
|
End Sub
|
|
Friend Async Function RemoveAt(ByVal Index As Integer) As Task
|
|
If Index.ValueBetween(0, Count - 1) Then
|
|
With Plans(Index)
|
|
.Stop()
|
|
If .Working Then
|
|
Await Task.Run(Sub()
|
|
While .Working : Thread.Sleep(510) : End While
|
|
End Sub)
|
|
End If
|
|
.Dispose()
|
|
End With
|
|
Plans.RemoveAt(Index)
|
|
Plans.ListReindex
|
|
Update()
|
|
End If
|
|
End Function
|
|
Private _UpdateRequired As Boolean = False
|
|
Friend Sub Update()
|
|
_UpdateRequired = True
|
|
Try
|
|
If Plans.Count > 0 Then
|
|
Using x As New XmlFile With {.Name = "Scheduler", .AllowSameNames = True} : x.AddRange(Plans) : x.Save(File) : End Using
|
|
Else
|
|
File.Delete()
|
|
End If
|
|
_UpdateRequired = False
|
|
Catch
|
|
End Try
|
|
End Sub
|
|
#Region "Groups Support"
|
|
Friend Sub GROUPS_Updated(ByVal Sender As DownloadGroup)
|
|
If Count > 0 Then Plans.ForEach(Sub(p) p.GROUPS_Updated(Sender))
|
|
End Sub
|
|
Friend Sub GROUPS_Deleted(ByVal Sender As DownloadGroup)
|
|
If Count > 0 Then Plans.ForEach(Sub(p) p.GROUPS_Deleted(Sender))
|
|
End Sub
|
|
#End Region
|
|
#Region "Execution"
|
|
Friend Async Function Start(ByVal Init As Boolean) As Task
|
|
Await Task.Run(Sub()
|
|
If Count > 0 Then
|
|
If Plans.Exists(PlanDownloading) Then PlansWaiter(PlanDownloading)
|
|
For Each Plan In Plans
|
|
Plan.Start(Init)
|
|
PlansWaiter(PlanDownloading)
|
|
Thread.Sleep(1000)
|
|
Next
|
|
End If
|
|
End Sub)
|
|
End Function
|
|
Friend Sub [Stop]()
|
|
If Count > 0 Then Plans.ForEach(Sub(p) p.Stop())
|
|
End Sub
|
|
Friend Property Pause(Optional ByVal LimitDate As Date? = Nothing) As PauseModes
|
|
Get
|
|
If Count > 0 Then Return Plans.FirstOrDefault(Function(p) p.Pause >= PauseModes.Disabled).Pause Else Return PauseModes.Disabled
|
|
End Get
|
|
Set(ByVal p As PauseModes)
|
|
If Count > 0 Then Plans.ForEach(Sub(pp) pp.Pause(LimitDate) = p)
|
|
End Set
|
|
End Property
|
|
#End Region
|
|
#Region "IEnumerable Support"
|
|
Private Function GetEnumerator() As IEnumerator(Of AutoDownloader) Implements IEnumerable(Of AutoDownloader).GetEnumerator
|
|
Return New MyEnumerator(Of AutoDownloader)(Me)
|
|
End Function
|
|
Private Function IEnumerable_GetEnumerator() As IEnumerator Implements IEnumerable.GetEnumerator
|
|
Return GetEnumerator()
|
|
End Function
|
|
#End Region
|
|
#Region "IDisposable Support"
|
|
Private disposedValue As Boolean = False
|
|
Protected Overridable Overloads Sub Dispose(ByVal disposing As Boolean)
|
|
If Not disposedValue Then
|
|
If disposing Then
|
|
[Stop]()
|
|
If Plans.Exists(PlanWorking) Then Task.WaitAll(Task.Run(Sub() PlansWaiter(PlanWorking)))
|
|
If _UpdateRequired Then Update()
|
|
Plans.ListClearDispose
|
|
End If
|
|
disposedValue = True
|
|
End If
|
|
End Sub
|
|
Protected Overrides Sub Finalize()
|
|
Dispose(False)
|
|
MyBase.Finalize()
|
|
End Sub
|
|
Friend Overloads Sub Dispose() Implements IDisposable.Dispose
|
|
Dispose(True)
|
|
GC.SuppressFinalize(Me)
|
|
End Sub
|
|
#End Region
|
|
End Class
|
|
End Namespace |