geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-11-10 05:35:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

2023.4.28.0

Browse Source 
Plugins
IPluginContentProvider: added DownloadSingleObject function; added tokens to GetMedia and Download functions; removed GetSpecialData function
Add IDownloadableMedia interface
Removed 'Channel' option from all functions and enums
ISiteSettings: added GetSingleMediaInstance function
ExchangeOptions: removed 'IsChannel'
UserMediaTypes: added Audio and AudioPre enums
IUserMedia, PluginUserMedia: changed ContentType and DownloadState from integers to their enums

SCrawler
Add YouTube standalone downloader
Add gallery-dl & yt-dlp support
Remove 'UserInfo' requirement from 'ProfilesSaved'
Update 'SiteSettingsBase' to use domains and Netscape cookies
UserDataBase: remove channels; remove old 'Merge' const; standardize SavedPosts file naming; move 'ValidateMD5' function from Twitter to UserDataBase to use it in other UserData classes; add 'DownloadSingleObject' environment for single posts; add validating file extension for m3u8 during download; add reindex of video file during download

Rewritten DomainsContainer
Create a universal settings form and PSettingsArttribute
Gfycat, Imgur: turn these classes into IUserData to download a single object

All plugins: update 'GetInstance' function for saved posts; update domains where implemented; remove 'OptionForm' where it exists; update options where they exist; update unix date providers; reconfigure channels where they exist

LPSG: fix attachments; update converters and regex
Add sites: ThisVid, Mastodon, Pinterest, YouTube, YouTube music
Reddit: standardize container parsing for all data types; new channel environment; fix 'ReparseMissing' function; redirect data downloading to the base download function, saved crossposts support
Twitter: fixed gif path bug; fixed downloading saved posts
PornHub: hide unnecessary errors; photo galleries bug
RedGifs: add 'UserAgent' option

Added icons to download progress

Rename some objects
Completely redesigned standalone downloader form and rewritten its environment
WebClient2: update to use tokens

Labels: update label form (save labels to file only when OK button is clicked); change removing labels.txt from recycle bin to permanent; disable storing label 'NoParsedUser'

UserCreatorForm: remove the 'Channel' checkbox and related functions; ability to extract the user's URL from the buffer and apply parameters if found
Remove temporary 'EncryptCookies' module

MainFrame: added simplified way to create new users (Ctrl+Insert to create a new user with default parameters from clipboard URL); removed SCrawler command line argument "-v" (remove the ability to run SCrawler as video downloader)
PropertyValueHost: update for option forms compatibility
SettingsHost: removed 'GetSpecialData' fork; added 'GetSingleMediaInstance' fork
UserDataHost: update functions with tokens; update events; add 'DownloadSingleObject' function
Settings: add the ability to get environment from 4 destinations; add the ability to set the program environment manually; add CMDEncoding; add cache; remove the old function 'RemoveUnusedPlugins'; add 'STDownloader' properties; add YT compatibility; add new notification options; add deleting user settings file when 'SettingsCLS.Dispose()' if where are no users in SCrawler
UserFinder: remove old 'Merge' const; remove channel option
UserInfo: remove channel option
This commit is contained in:
Andy
2023-04-28 10:13:46 +03:00
parent db9e2cfb88
commit b2a9b22478
270 changed files with 18120 additions and 3332 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
SCrawler/Download/DownloadProgress.vb
View File

@@ -24,6 +24,7 @@ Namespace DownloadObjects
Private WithEvents BTT_OPEN As Button
Private ReadOnly PR_MAIN As ProgressBar
Private ReadOnly LBL_INFO As Label
Private ReadOnly Icon As PictureBox
#End Region
Private ReadOnly Property Instance As API.Base.ProfileSaved
Friend ReadOnly Property Job As TDJob
@@ -38,7 +39,21 @@ Namespace DownloadObjects
TP_CONTROLS = New TableLayoutPanel With {.Margin = New Padding(0), .Dock = DockStyle.Fill}
PR_MAIN = New ProgressBar With {.Dock = DockStyle.Fill}
LBL_INFO = New Label With {.Text = String.Empty, .Dock = DockStyle.Fill}
Icon = New PictureBox With {
.SizeMode = PictureBoxSizeMode.Zoom,
.Dock = DockStyle.Fill,
.Margin = New Padding(3),
.Padding = New Padding(3)
}
CreateButton(BTT_STOP, My.Resources.DeletePic_24)
Dim img As Image = Nothing
If Not _Job.Host Is Nothing Then
With Job.Host.Source
If Not .Icon Is Nothing Then img = .Icon.ToBitmap
If img Is Nothing AndAlso Not .Image Is Nothing Then img = .Image
End With
End If
If Not img Is Nothing Then Icon.Image = img : Icon.InitialImage = img
If Job.Type = Download.Main Then
LBL_INFO.Margin = New Padding(3)
@@ -48,6 +63,7 @@ Namespace DownloadObjects
.RowCount = 1
End With
With TP_CONTROLS
.ColumnStyles.Add(New ColumnStyle(SizeType.Absolute, 30))
.ColumnStyles.Add(New ColumnStyle(SizeType.Absolute, 30))
.ColumnStyles.Add(New ColumnStyle(SizeType.Absolute, 150))
.ColumnStyles.Add(New ColumnStyle(SizeType.Percent, 100))
@@ -55,9 +71,10 @@ Namespace DownloadObjects
.RowStyles.Add(New RowStyle(SizeType.Percent, 100))
.RowCount = 1
With .Controls
.Add(BTT_STOP, 0, 0)
.Add(PR_MAIN, 1, 0)
.Add(LBL_INFO, 2, 0)
If Not img Is Nothing Then .Add(Icon, 0, 0)
.Add(BTT_STOP, 1, 0)
.Add(PR_MAIN, 2, 0)
.Add(LBL_INFO, 3, 0)
End With
End With
TP_MAIN.Controls.Add(TP_CONTROLS, 0, 0)
@@ -71,16 +88,18 @@ Namespace DownloadObjects
.Add(New ColumnStyle(SizeType.Absolute, 30))
.Add(New ColumnStyle(SizeType.Absolute, 30))
.Add(New ColumnStyle(SizeType.Absolute, 30))
.Add(New ColumnStyle(SizeType.Absolute, 30))
.Add(New ColumnStyle(SizeType.Percent, 100))
End With
.ColumnCount = 4
.ColumnCount = .ColumnStyles.Count
.RowStyles.Add(New RowStyle(SizeType.Percent, 50))
.RowCount = 1
With .Controls
.Add(BTT_START, 0, 0)
.Add(BTT_STOP, 1, 0)
.Add(BTT_OPEN, 2, 0)
.Add(PR_MAIN, 3, 0)
If Not img Is Nothing Then .Add(Icon, 0, 0)
.Add(BTT_START, 1, 0)
.Add(BTT_STOP, 2, 0)
.Add(BTT_OPEN, 3, 0)
.Add(PR_MAIN, 4, 0)
End With
End With
With TP_MAIN
@@ -129,11 +148,13 @@ Namespace DownloadObjects
End Sub
#End Region
#Region "Start, Stop"
Friend Sub Start()
Job.Start(AddressOf DownloadData)
Private _IsMultiple As Boolean = False
Friend Sub Start(Optional ByVal Multiple As Boolean = False)
_IsMultiple = Multiple
Job.StartThread(AddressOf DownloadData)
End Sub
Friend Sub [Stop]()
Job.Stop()
Job.Cancel()
End Sub
#End Region
#Region "SavedPosts downloading"
@@ -144,15 +165,16 @@ Namespace DownloadObjects
btte.Invoke(BTT_STOP, True)
Job.Progress.InformationTemporary = $"{Job.Host.Name} downloading started"
Job.Start()
Instance.Download(Job.Token)
Instance.Download(Job.Token, _IsMultiple)
RaiseEvent DownloadDone(SettingsCLS.NotificationObjects.SavedPosts, $"Downloading saved {Job.Host.Name} posts is completed")
Catch ex As Exception
Job.Progress.InformationTemporary = $"{Job.Host.Name} downloading error"
ErrorsDescriber.Execute(EDP.LogMessageValue, ex, {$"{Job.Host.Name} saved posts downloading error", "Saved posts"})
Finally
_IsMultiple = False
btte.Invoke(BTT_START, True)
btte.Invoke(BTT_STOP, False)
Job.Stopped()
Job.Finish()
If Job.Type = Download.SavedPosts Then Job.Progress.Maximum = 0 : Job.Progress.Value = 0
End Try
End Sub
@@ -173,6 +195,7 @@ Namespace DownloadObjects
If Not BTT_START Is Nothing Then BTT_START.Dispose()
If Not BTT_STOP Is Nothing Then BTT_STOP.Dispose()
If Not BTT_OPEN Is Nothing Then BTT_OPEN.Dispose()
If Not Icon Is Nothing Then Icon.Dispose()
PR_MAIN.Dispose()
LBL_INFO.Dispose()
TP_CONTROLS.Controls.Clear()