geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

2023.08.19.0

Browse Source 
API.Base: update 'TokenBatch', 'GDLBatch'; add 'YTDLPBatch'
API.Base.UserDataBase: update 'EraseData' function
API.Redgifs: fix hd/sd issue
API.TikTok: update dates in commands; replace 'TokenBatch' with 'YTDLPBatch'
UserDownloadQueueForm: update functions
This commit is contained in:
Andy
2023-08-19 18:42:01 +03:00
parent 28ae44f0ae
commit f0014d2874
9 changed files with 76 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
SCrawler/API/Base/GDLBatch.vb → SCrawler/API/Base/GDL.vb
View File

@@ -67,7 +67,6 @@ Namespace API.Base.GDL
End Function
End Module
Friend Class GDLBatch : Inherits TokenBatch
Friend Property TempPostsList As List(Of String)
Friend Const UrlLibStart As String = "[urllib3.connectionpool][debug]"
Friend Const UrlTextStart As String = UrlLibStart & " https"
Friend Sub New(ByVal _Token As Threading.CancellationToken)
@@ -75,20 +74,11 @@ Namespace API.Base.GDL
MainProcessName = "gallery-dl"
ChangeDirectory(Settings.GalleryDLFile.File)
End Sub
Public Overrides Sub Create()
If TempPostsList Is Nothing Then TempPostsList = New List(Of String)
MyBase.Create()
End Sub
Protected Overrides Async Sub OutputDataReceiver(ByVal Sender As Object, ByVal e As DataReceivedEventArgs)
If Not ProcessKilled Then
MyBase.OutputDataReceiver(Sender, e)
Await Validate(e.Data)
End If
End Sub
Protected Overridable Async Function Validate(ByVal Value As String) As Task
If Not ProcessKilled AndAlso Await Task.Run(Of Boolean)(Function() Token.IsCancellationRequested OrElse
(Not Value.IsEmptyString AndAlso
TempPostsList.Exists(Function(v) Value.Contains(v)))) Then Kill()
End Function
End Class
End Namespace

10
SCrawler/API/Base/TokenBatch.vb
View File

@@ -10,11 +10,16 @@ Imports System.Threading
Imports PersonalUtilities.Tools
Namespace API.Base
Friend Class TokenBatch : Inherits BatchExecutor
Friend Property TempPostsList As List(Of String)
Protected ReadOnly Token As CancellationToken
Friend Sub New(ByVal _Token As CancellationToken)
MyBase.New(True)
Token = _Token
End Sub
Public Overrides Sub Create()
If TempPostsList Is Nothing Then TempPostsList = New List(Of String)
MyBase.Create()
End Sub
Protected Overrides Async Sub OutputDataReceiver(ByVal Sender As Object, ByVal e As DataReceivedEventArgs)
MyBase.OutputDataReceiver(Sender, e)
Await Task.Run(Sub() If Token.IsCancellationRequested Then Kill())
@@ -23,5 +28,10 @@ Namespace API.Base
MyBase.ErrorDataReceiver(Sender, e)
Await Task.Run(Sub() If Token.IsCancellationRequested Then Kill())
End Sub
Protected Overridable Async Function Validate(ByVal Value As String) As Task
If Not ProcessKilled AndAlso Await Task.Run(Of Boolean)(Function() Token.IsCancellationRequested OrElse
(Not Value.IsEmptyString AndAlso
TempPostsList.Exists(Function(v) Value.Contains(v)))) Then Kill()
End Function
End Class
End Namespace

12
SCrawler/API/Base/UserDataBase.vb
View File

@@ -1829,12 +1829,6 @@ BlockNullPicture:
If m.Contains(IUserData.EraseMode.History) Then
If MyFilePosts.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True
If MyFileData.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True
If result Then
_TempPostsList.Clear()
_TempMediaList.Clear()
_ContentNew.Clear()
_ContentList.Clear()
End If
End If
If m.Contains(IUserData.EraseMode.Data) Then
Dim files As List(Of SFile) = SFile.GetFiles(DownloadContentDefault_GetRootDir.CSFileP,, SearchOption.AllDirectories, e)
@@ -1843,6 +1837,12 @@ BlockNullPicture:
LatestData.Clear()
result = True
End If
If result Then
_TempPostsList.Clear()
_TempMediaList.Clear()
_ContentNew.Clear()
_ContentList.Clear()
End If
End If
End If
Return result

18
SCrawler/API/Base/YTDLP.vb Normal file
View File

@@ -0,0 +1,18 @@
' Copyright (C) 2023 Andy https://github.com/AAndyProgram
' This program is free software: you can redistribute it and/or modify
' it under the terms of the GNU General Public License as published by
' the Free Software Foundation, either version 3 of the License, or
' (at your option) any later version.
'
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Namespace API.Base.YTDLP
Friend Class YTDLPBatch : Inherits GDL.GDLBatch
Friend Sub New(ByVal _Token As Threading.CancellationToken)
MyBase.New(_Token)
Commands.Clear()
MainProcessName = "yt-dlp"
ChangeDirectory(Settings.YtdlpFile.File)
End Sub
End Class
End Namespace

2
SCrawler/API/Redgifs/UserData.vb
View File

@@ -83,7 +83,7 @@ Namespace API.RedGifs
If Not j Is Nothing Then
With j("urls")
If .ListExists Then
Dim u$ = If(.Item("hd"), .Item("sd")).XmlIfNothingValue
Dim u$ = .Value("hd").IfNullOrEmpty(.Value("sd"))
If Not u.IsEmptyString Then
Dim ut As UTypes = UTypes.Undefined
'Type 1: video

17
SCrawler/API/TikTok/UserData.vb
View File

@@ -71,6 +71,7 @@ Namespace API.TikTok
TitleUseNative = .Value(Name_TitleUseNative).FromXML(Of Boolean)(True)
TitleAddVideoID = .Value(Name_TitleAddVideoID).FromXML(Of Boolean)(True)
LastDownloadDate = AConvert(Of Date)(.Value(Name_LastDownloadDate), ADateTime.Formats.BaseDateTime, Nothing)
If Not LastDownloadDate.HasValue Then LastDownloadDate = LastUpdated
Else
.Add(Name_RemoveTagsFromTitle, RemoveTagsFromTitle.BoolToInteger)
.Add(Name_TitleUseNative, TitleUseNative.BoolToInteger)
@@ -109,12 +110,20 @@ Namespace API.TikTok
End With
If LastDownloadDate.HasValue Then
If dateAfter.HasValue And Not DownloadDateFrom.HasValue Then
If Not DownloadDateTo.HasValue And Not DownloadDateFrom.HasValue Then
If LastDownloadDate.Value.AddDays(1) <= Now Then
dateAfter = LastDownloadDate.Value
Else
dateAfter = LastDownloadDate.Value.AddDays(-1)
End If
dateBefore = Nothing
ElseIf dateAfter.HasValue And Not DownloadDateFrom.HasValue Then
If (LastDownloadDate.Value - dateAfter.Value).TotalDays > 1 Then dateAfter = dateAfter.Value.AddDays(1)
End If
End If
Using b As New TokenBatch(Token)
Using b As New YTDLP.YTDLPBatch(Token) With {.TempPostsList = _TempPostsList}
b.Commands.Clear()
b.ChangeDirectory(cache)
b.Encoding = BatchExecutor.UnicodeEncoding
b.Execute(CreateYTCommand(cache.RootDirectory, URL, False, dateBefore, dateAfter))
@@ -163,6 +172,7 @@ Namespace API.TikTok
End If
Next
End If
If _TempMediaList.Count > 0 Then LastDownloadDate = Now
Catch ex As Exception
ProcessException(ex, Token, $"data downloading error [{URL}]")
End Try
@@ -210,6 +220,9 @@ Namespace API.TikTok
command &= "-o %(id)s"
End If
End If
'#If DEBUG Then
'Debug.WriteLine(command)
'#End If
Return command
End Function
#End Region

40
SCrawler/Download/UserDownloadQueueForm.vb
View File

@@ -77,10 +77,13 @@ Namespace DownloadObjects
If b Then e.Handled = True
End Sub
Friend Sub Downloader_Downloading(ByVal Value As Boolean)
If Not Value Then ControlInvokeFast(LIST_QUEUE, Sub()
LIST_QUEUE.Items.Clear()
Tokens.ListClearDispose
End Sub, EDP.None)
Try
If Not Value Then ControlInvokeFast(LIST_QUEUE, Sub()
LIST_QUEUE.Items.Clear()
Tokens.ListClearDispose
End Sub, EDP.None)
Catch
End Try
End Sub
Friend Sub Downloader_UserDownloadStateChanged(ByVal User As IUserData, ByVal IsDownloading As Boolean)
Try
@@ -93,8 +96,8 @@ Namespace DownloadObjects
LIST_QUEUE.Items.Remove(u)
End If
LIST_QUEUE.Refresh()
End Sub)
Catch ex As Exception
End Sub, EDP.None)
Catch
End Try
End Sub
Private Sub User_UserDownloadStateChanged(ByVal User As IUserData, ByVal IsDownloading As Boolean)
@@ -111,7 +114,7 @@ Namespace DownloadObjects
LIST_QUEUE.Refresh()
End If
End If
End Sub)
End Sub, EDP.None)
Catch
End Try
End Sub
@@ -143,22 +146,23 @@ Namespace DownloadObjects
Tokens.Add(token)
End If
End If
Catch ex As Exception
Catch
End Try
End Sub
Private Sub FindUser()
Try
MainFrameObj.FocusUser(GetUserSelectedUser().Key, True)
Catch ex As Exception
End Try
Try : MainFrameObj.FocusUser(GetUserSelectedUser().Key, True) : Catch : End Try
End Sub
Private Function GetUserSelectedUser() As ListUser
Dim lu As ListUser = Nothing
ControlInvokeFast(LIST_QUEUE, Sub()
Dim sIndx% = LIST_QUEUE.SelectedIndex
If sIndx >= 0 Then lu = LIST_QUEUE.Items(sIndx)
End Sub)
Return lu
Try
Dim lu As ListUser = Nothing
ControlInvokeFast(LIST_QUEUE, Sub()
Dim sIndx% = LIST_QUEUE.SelectedIndex
If sIndx >= 0 Then lu = LIST_QUEUE.Items(sIndx)
End Sub, EDP.None)
Return lu
Catch
Return Nothing
End Try
End Function
End Class
End Namespace

4
SCrawler/My Project/AssemblyInfo.vb
View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("2023.8.17.0")>
<Assembly: AssemblyFileVersion("2023.8.17.0")>
<Assembly: AssemblyVersion("2023.8.19.0")>
<Assembly: AssemblyFileVersion("2023.8.19.0")>
<Assembly: NeutralResourcesLanguage("en")>

3
SCrawler/SCrawler.vbproj
View File

@@ -168,13 +168,14 @@
<Compile Include="API\Base\Declarations.vb" />
<Compile Include="API\Base\DeclaredNames.vb" />
<Compile Include="API\Base\DownDetector.vb" />
<Compile Include="API\Base\GDLBatch.vb" />
<Compile Include="API\Base\GDL.vb" />
<Compile Include="API\Base\IUserData.vb" />
<Compile Include="API\Base\M3U8Base.vb" />
<Compile Include="API\Base\ProfileSaved.vb" />
<Compile Include="API\Base\SiteSettingsBase.vb" />
<Compile Include="API\Base\Structures.vb" />
<Compile Include="API\Base\TokenBatch.vb" />
<Compile Include="API\Base\YTDLP.vb" />
<Compile Include="API\Instagram\EditorExchangeOptions.vb" />
<Compile Include="API\JustForFans\Declarations.vb" />
<Compile Include="API\JustForFans\M3U8.vb" />