mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-16 00:32:18 +00:00
2023.4.28.0
Plugins IPluginContentProvider: added DownloadSingleObject function; added tokens to GetMedia and Download functions; removed GetSpecialData function Add IDownloadableMedia interface Removed 'Channel' option from all functions and enums ISiteSettings: added GetSingleMediaInstance function ExchangeOptions: removed 'IsChannel' UserMediaTypes: added Audio and AudioPre enums IUserMedia, PluginUserMedia: changed ContentType and DownloadState from integers to their enums SCrawler Add YouTube standalone downloader Add gallery-dl & yt-dlp support Remove 'UserInfo' requirement from 'ProfilesSaved' Update 'SiteSettingsBase' to use domains and Netscape cookies UserDataBase: remove channels; remove old 'Merge' const; standardize SavedPosts file naming; move 'ValidateMD5' function from Twitter to UserDataBase to use it in other UserData classes; add 'DownloadSingleObject' environment for single posts; add validating file extension for m3u8 during download; add reindex of video file during download Rewritten DomainsContainer Create a universal settings form and PSettingsArttribute Gfycat, Imgur: turn these classes into IUserData to download a single object All plugins: update 'GetInstance' function for saved posts; update domains where implemented; remove 'OptionForm' where it exists; update options where they exist; update unix date providers; reconfigure channels where they exist LPSG: fix attachments; update converters and regex Add sites: ThisVid, Mastodon, Pinterest, YouTube, YouTube music Reddit: standardize container parsing for all data types; new channel environment; fix 'ReparseMissing' function; redirect data downloading to the base download function, saved crossposts support Twitter: fixed gif path bug; fixed downloading saved posts PornHub: hide unnecessary errors; photo galleries bug RedGifs: add 'UserAgent' option Added icons to download progress Rename some objects Completely redesigned standalone downloader form and rewritten its environment WebClient2: update to use tokens Labels: update label form (save labels to file only when OK button is clicked); change removing labels.txt from recycle bin to permanent; disable storing label 'NoParsedUser' UserCreatorForm: remove the 'Channel' checkbox and related functions; ability to extract the user's URL from the buffer and apply parameters if found Remove temporary 'EncryptCookies' module MainFrame: added simplified way to create new users (Ctrl+Insert to create a new user with default parameters from clipboard URL); removed SCrawler command line argument "-v" (remove the ability to run SCrawler as video downloader) PropertyValueHost: update for option forms compatibility SettingsHost: removed 'GetSpecialData' fork; added 'GetSingleMediaInstance' fork UserDataHost: update functions with tokens; update events; add 'DownloadSingleObject' function Settings: add the ability to get environment from 4 destinations; add the ability to set the program environment manually; add CMDEncoding; add cache; remove the old function 'RemoveUnusedPlugins'; add 'STDownloader' properties; add YT compatibility; add new notification options; add deleting user settings file when 'SettingsCLS.Dispose()' if where are no users in SCrawler UserFinder: remove old 'Merge' const; remove channel option UserInfo: remove channel option
This commit is contained in:
@@ -6,12 +6,12 @@
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports PersonalUtilities.Functions.XML
|
||||
Imports PersonalUtilities.Functions.XML.Base
|
||||
Imports PersonalUtilities.Tools
|
||||
Imports System.Threading
|
||||
Imports SCrawler.API.Base
|
||||
Imports SCrawler.Plugin.Hosts
|
||||
Imports PersonalUtilities.Tools
|
||||
Imports PersonalUtilities.Functions.XML
|
||||
Imports PersonalUtilities.Functions.XML.Base
|
||||
Imports Download = SCrawler.Plugin.ISiteSettings.Download
|
||||
Namespace DownloadObjects
|
||||
Friend Class TDownloader : Implements IDisposable
|
||||
@@ -101,7 +101,7 @@ Namespace DownloadObjects
|
||||
End Using
|
||||
End If
|
||||
Catch ex As Exception
|
||||
ErrorsDescriber.Execute(EDP.SendInLog, ex, "[DownloadObjects.TDownloader.FilesSave]")
|
||||
ErrorsDescriber.Execute(EDP.SendToLog, ex, "[DownloadObjects.TDownloader.FilesSave]")
|
||||
End Try
|
||||
End Sub
|
||||
Private Sub FilesBackupLastSession()
|
||||
@@ -109,7 +109,7 @@ Namespace DownloadObjects
|
||||
If Not FilesLastSessionBackedup Then
|
||||
If FilesSessionActual.Exists Then
|
||||
If FilesSessionBackup.Exists Then
|
||||
Dim f As SFile = SFile.Indexed_IndexFile(FilesSessionBackup)
|
||||
Dim f As SFile = SFile.IndexReindex(FilesSessionBackup)
|
||||
SFile.Rename(FilesSessionBackup, f)
|
||||
RemoveLogFiles(FilesSessionBackup, 10)
|
||||
FilesSessionBackup.Delete()
|
||||
@@ -118,7 +118,7 @@ Namespace DownloadObjects
|
||||
End If
|
||||
End If
|
||||
Catch ex As Exception
|
||||
ErrorsDescriber.Execute(EDP.SendInLog, ex, "[DownloadObjects.TDownloader.FilesBackupLastSession]")
|
||||
ErrorsDescriber.Execute(EDP.SendToLog, ex, "[DownloadObjects.TDownloader.FilesBackupLastSession]")
|
||||
Finally
|
||||
FilesLastSessionBackedup = True
|
||||
End Try
|
||||
@@ -244,7 +244,7 @@ Namespace DownloadObjects
|
||||
Token = TokenSource.Token
|
||||
_Working = True
|
||||
End Sub
|
||||
Public Overrides Sub Stopped()
|
||||
Public Overrides Sub Finish()
|
||||
_Working = False
|
||||
TokenSource = Nothing
|
||||
Try
|
||||
@@ -331,7 +331,7 @@ Namespace DownloadObjects
|
||||
MyProgressForm.DisableProgressChange = False
|
||||
Do While Pool.Exists(Function(p) p.Count > 0 Or p.Working)
|
||||
For Each j As Job In Pool
|
||||
If j.Count > 0 And Not j.Working And Not Suspended Then j.Start(New ThreadStart(Sub() StartDownloading(j)))
|
||||
If j.Count > 0 And Not j.Working And Not Suspended Then j.StartThread(New ThreadStart(Sub() StartDownloading(j)))
|
||||
Next
|
||||
Thread.Sleep(200)
|
||||
Loop
|
||||
@@ -384,14 +384,14 @@ Namespace DownloadObjects
|
||||
_Job.Progress.InformationTemporary = pt("Downloading canceled")
|
||||
Catch ex As Exception
|
||||
_Job.Progress.InformationTemporary = pt("Downloading error")
|
||||
ErrorsDescriber.Execute(EDP.SendInLog, ex, "TDownloader.Start")
|
||||
ErrorsDescriber.Execute(EDP.SendToLog, ex, "TDownloader.Start")
|
||||
Finally
|
||||
If _Job.Count > 0 Then _Job.Clear()
|
||||
_Job.Stopped()
|
||||
_Job.Finish()
|
||||
End Try
|
||||
End Sub
|
||||
Friend Sub [Stop]()
|
||||
If Pool.Count > 0 Then Pool.ForEach(Sub(j) If j.Working Then j.Stop())
|
||||
If Pool.Count > 0 Then Pool.ForEach(Sub(j) If j.Working Then j.Cancel())
|
||||
End Sub
|
||||
Private Sub UpdateJobsLabel()
|
||||
RaiseEvent JobsChange(Count)
|
||||
@@ -456,10 +456,10 @@ Namespace DownloadObjects
|
||||
End If
|
||||
End If
|
||||
Catch aoex As ArgumentOutOfRangeException
|
||||
ErrorsDescriber.Execute(EDP.SendInLog, aoex, $"TDownloader.DownloadData: index out of range ({_Job.Count})")
|
||||
ErrorsDescriber.Execute(EDP.SendToLog, aoex, $"TDownloader.DownloadData: index out of range ({_Job.Count})")
|
||||
Catch oex As OperationCanceledException When _Job.IsCancellationRequested
|
||||
Catch ex As Exception
|
||||
ErrorsDescriber.Execute(EDP.SendInLog, ex, "TDownloader.DownloadData")
|
||||
ErrorsDescriber.Execute(EDP.SendToLog, ex, "TDownloader.DownloadData")
|
||||
Finally
|
||||
If Settings.UserListUpdateRequired Then _
|
||||
Task.WaitAll(Task.Run(Sub()
|
||||
|
||||
Reference in New Issue
Block a user