2023.08.19.0

API.Base: update 'TokenBatch', 'GDLBatch'; add 'YTDLPBatch'
API.Base.UserDataBase: update 'EraseData' function
API.Redgifs: fix hd/sd issue
API.TikTok: update dates in commands; replace 'TokenBatch' with 'YTDLPBatch'
UserDownloadQueueForm: update functions

SCrawler/API/Base/GDL.vb

@@ -67,7 +67,6 @@ Namespace API.Base.GDL
         End Function
     End Module
     Friend Class GDLBatch : Inherits TokenBatch
-        Friend Property TempPostsList As List(Of String)
         Friend Const UrlLibStart As String = "[urllib3.connectionpool][debug]"
         Friend Const UrlTextStart As String = UrlLibStart & " https"
         Friend Sub New(ByVal _Token As Threading.CancellationToken)
@@ -75,20 +74,11 @@ Namespace API.Base.GDL
             MainProcessName = "gallery-dl"
             ChangeDirectory(Settings.GalleryDLFile.File)
         End Sub
-        Public Overrides Sub Create()
-            If TempPostsList Is Nothing Then TempPostsList = New List(Of String)
-            MyBase.Create()
-        End Sub
         Protected Overrides Async Sub OutputDataReceiver(ByVal Sender As Object, ByVal e As DataReceivedEventArgs)
             If Not ProcessKilled Then
                 MyBase.OutputDataReceiver(Sender, e)
                 Await Validate(e.Data)
             End If
         End Sub
-        Protected Overridable Async Function Validate(ByVal Value As String) As Task
-            If Not ProcessKilled AndAlso Await Task.Run(Of Boolean)(Function() Token.IsCancellationRequested OrElse
-                                                                               (Not Value.IsEmptyString AndAlso
-                                                                               TempPostsList.Exists(Function(v) Value.Contains(v)))) Then Kill()
-        End Function
     End Class
 End Namespace

SCrawler/API/Base/TokenBatch.vb

@@ -10,11 +10,16 @@ Imports System.Threading
 Imports PersonalUtilities.Tools
 Namespace API.Base
     Friend Class TokenBatch : Inherits BatchExecutor
+        Friend Property TempPostsList As List(Of String)
         Protected ReadOnly Token As CancellationToken
         Friend Sub New(ByVal _Token As CancellationToken)
             MyBase.New(True)
             Token = _Token
         End Sub
+        Public Overrides Sub Create()
+            If TempPostsList Is Nothing Then TempPostsList = New List(Of String)
+            MyBase.Create()
+        End Sub
         Protected Overrides Async Sub OutputDataReceiver(ByVal Sender As Object, ByVal e As DataReceivedEventArgs)
             MyBase.OutputDataReceiver(Sender, e)
             Await Task.Run(Sub() If Token.IsCancellationRequested Then Kill())
@@ -23,5 +28,10 @@ Namespace API.Base
             MyBase.ErrorDataReceiver(Sender, e)
             Await Task.Run(Sub() If Token.IsCancellationRequested Then Kill())
         End Sub
+        Protected Overridable Async Function Validate(ByVal Value As String) As Task
+            If Not ProcessKilled AndAlso Await Task.Run(Of Boolean)(Function() Token.IsCancellationRequested OrElse
+                                                                               (Not Value.IsEmptyString AndAlso
+                                                                               TempPostsList.Exists(Function(v) Value.Contains(v)))) Then Kill()
+        End Function
     End Class
 End Namespace

SCrawler/API/Base/UserDataBase.vb

@@ -1829,12 +1829,6 @@ BlockNullPicture:
                         If m.Contains(IUserData.EraseMode.History) Then
                             If MyFilePosts.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True
                             If MyFileData.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True
-                            If result Then
-                                _TempPostsList.Clear()
-                                _TempMediaList.Clear()
-                                _ContentNew.Clear()
-                                _ContentList.Clear()
-                            End If
                         End If
                         If m.Contains(IUserData.EraseMode.Data) Then
                             Dim files As List(Of SFile) = SFile.GetFiles(DownloadContentDefault_GetRootDir.CSFileP,, SearchOption.AllDirectories, e)
@@ -1843,6 +1837,12 @@ BlockNullPicture:
                             LatestData.Clear()
                             result = True
                         End If
+                        If result Then
+                            _TempPostsList.Clear()
+                            _TempMediaList.Clear()
+                            _ContentNew.Clear()
+                            _ContentList.Clear()
+                        End If
                     End If
                 End If
                 Return result

SCrawler/API/Base/YTDLP.vb

@@ -0,0 +1,18 @@
+' Copyright (C) 2023  Andy https://github.com/AAndyProgram
+' This program is free software: you can redistribute it and/or modify
+' it under the terms of the GNU General Public License as published by
+' the Free Software Foundation, either version 3 of the License, or
+' (at your option) any later version.
+'
+' This program is distributed in the hope that it will be useful,
+' but WITHOUT ANY WARRANTY
+Namespace API.Base.YTDLP
+    Friend Class YTDLPBatch : Inherits GDL.GDLBatch
+        Friend Sub New(ByVal _Token As Threading.CancellationToken)
+            MyBase.New(_Token)
+            Commands.Clear()
+            MainProcessName = "yt-dlp"
+            ChangeDirectory(Settings.YtdlpFile.File)
+        End Sub
+    End Class
+End Namespace

SCrawler/API/Redgifs/UserData.vb

@@ -83,7 +83,7 @@ Namespace API.RedGifs
             If Not j Is Nothing Then
                 With j("urls")
                     If .ListExists Then
-                        Dim u$ = If(.Item("hd"), .Item("sd")).XmlIfNothingValue
+                        Dim u$ = .Value("hd").IfNullOrEmpty(.Value("sd"))
                         If Not u.IsEmptyString Then
                             Dim ut As UTypes = UTypes.Undefined
                             'Type 1: video

SCrawler/API/TikTok/UserData.vb

@@ -71,6 +71,7 @@ Namespace API.TikTok
                     TitleUseNative = .Value(Name_TitleUseNative).FromXML(Of Boolean)(True)
                     TitleAddVideoID = .Value(Name_TitleAddVideoID).FromXML(Of Boolean)(True)
                     LastDownloadDate = AConvert(Of Date)(.Value(Name_LastDownloadDate), ADateTime.Formats.BaseDateTime, Nothing)
+                    If Not LastDownloadDate.HasValue Then LastDownloadDate = LastUpdated
                 Else
                     .Add(Name_RemoveTagsFromTitle, RemoveTagsFromTitle.BoolToInteger)
                     .Add(Name_TitleUseNative, TitleUseNative.BoolToInteger)
@@ -109,12 +110,20 @@ Namespace API.TikTok
                     End With
 
                     If LastDownloadDate.HasValue Then
-                        If dateAfter.HasValue And Not DownloadDateFrom.HasValue Then
+                        If Not DownloadDateTo.HasValue And Not DownloadDateFrom.HasValue Then
+                            If LastDownloadDate.Value.AddDays(1) <= Now Then
+                                dateAfter = LastDownloadDate.Value
+                            Else
+                                dateAfter = LastDownloadDate.Value.AddDays(-1)
+                            End If
+                            dateBefore = Nothing
+                        ElseIf dateAfter.HasValue And Not DownloadDateFrom.HasValue Then
                             If (LastDownloadDate.Value - dateAfter.Value).TotalDays > 1 Then dateAfter = dateAfter.Value.AddDays(1)
                         End If
                     End If
 
-                    Using b As New TokenBatch(Token)
+                    Using b As New YTDLP.YTDLPBatch(Token) With {.TempPostsList = _TempPostsList}
+                        b.Commands.Clear()
                         b.ChangeDirectory(cache)
                         b.Encoding = BatchExecutor.UnicodeEncoding
                         b.Execute(CreateYTCommand(cache.RootDirectory, URL, False, dateBefore, dateAfter))
@@ -163,6 +172,7 @@ Namespace API.TikTok
                             End If
                         Next
                     End If
+                    If _TempMediaList.Count > 0 Then LastDownloadDate = Now
                 Catch ex As Exception
                     ProcessException(ex, Token, $"data downloading error [{URL}]")
                 End Try
@@ -210,6 +220,9 @@ Namespace API.TikTok
                     command &= "-o %(id)s"
                 End If
             End If
+            '#If DEBUG Then
+            'Debug.WriteLine(command)
+            '#End If
             Return command
         End Function
 #End Region

SCrawler/Download/UserDownloadQueueForm.vb

@@ -77,10 +77,13 @@ Namespace DownloadObjects
             If b Then e.Handled = True
         End Sub
         Friend Sub Downloader_Downloading(ByVal Value As Boolean)
-            If Not Value Then ControlInvokeFast(LIST_QUEUE, Sub()
+            Try
-                                                                LIST_QUEUE.Items.Clear()
+                If Not Value Then ControlInvokeFast(LIST_QUEUE, Sub()
-                                                                Tokens.ListClearDispose
+                                                                    LIST_QUEUE.Items.Clear()
-                                                            End Sub, EDP.None)
+                                                                    Tokens.ListClearDispose
+                                                                End Sub, EDP.None)
+            Catch
+            End Try
         End Sub
         Friend Sub Downloader_UserDownloadStateChanged(ByVal User As IUserData, ByVal IsDownloading As Boolean)
             Try
@@ -93,8 +96,8 @@ Namespace DownloadObjects
                                                       LIST_QUEUE.Items.Remove(u)
                                                   End If
                                                   LIST_QUEUE.Refresh()
-                                              End Sub)
+                                              End Sub, EDP.None)
-            Catch ex As Exception
+            Catch
             End Try
         End Sub
         Private Sub User_UserDownloadStateChanged(ByVal User As IUserData, ByVal IsDownloading As Boolean)
@@ -111,7 +114,7 @@ Namespace DownloadObjects
                                               LIST_QUEUE.Refresh()
                                           End If
                                       End If
-                                  End Sub)
+                                  End Sub, EDP.None)
             Catch
             End Try
         End Sub
@@ -143,22 +146,23 @@ Namespace DownloadObjects
                         Tokens.Add(token)
                     End If
                 End If
-            Catch ex As Exception
+            Catch
             End Try
         End Sub
         Private Sub FindUser()
-            Try
+            Try : MainFrameObj.FocusUser(GetUserSelectedUser().Key, True) : Catch : End Try
-                MainFrameObj.FocusUser(GetUserSelectedUser().Key, True)
-            Catch ex As Exception
-            End Try
         End Sub
         Private Function GetUserSelectedUser() As ListUser
-            Dim lu As ListUser = Nothing
+            Try
-            ControlInvokeFast(LIST_QUEUE, Sub()
+                Dim lu As ListUser = Nothing
-                                              Dim sIndx% = LIST_QUEUE.SelectedIndex
+                ControlInvokeFast(LIST_QUEUE, Sub()
-                                              If sIndx >= 0 Then lu = LIST_QUEUE.Items(sIndx)
+                                                  Dim sIndx% = LIST_QUEUE.SelectedIndex
-                                          End Sub)
+                                                  If sIndx >= 0 Then lu = LIST_QUEUE.Items(sIndx)
-            Return lu
+                                              End Sub, EDP.None)
+                Return lu
+            Catch
+                Return Nothing
+            End Try
         End Function
     End Class
 End Namespace

SCrawler/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2023.8.17.0")>
+<Assembly: AssemblyVersion("2023.8.19.0")>
-<Assembly: AssemblyFileVersion("2023.8.17.0")>
+<Assembly: AssemblyFileVersion("2023.8.19.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/SCrawler.vbproj

@@ -168,13 +168,14 @@
     <Compile Include="API\Base\Declarations.vb" />
     <Compile Include="API\Base\DeclaredNames.vb" />
     <Compile Include="API\Base\DownDetector.vb" />
-    <Compile Include="API\Base\GDLBatch.vb" />
+    <Compile Include="API\Base\GDL.vb" />
     <Compile Include="API\Base\IUserData.vb" />
     <Compile Include="API\Base\M3U8Base.vb" />
     <Compile Include="API\Base\ProfileSaved.vb" />
     <Compile Include="API\Base\SiteSettingsBase.vb" />
     <Compile Include="API\Base\Structures.vb" />
     <Compile Include="API\Base\TokenBatch.vb" />
+    <Compile Include="API\Base\YTDLP.vb" />
     <Compile Include="API\Instagram\EditorExchangeOptions.vb" />
     <Compile Include="API\JustForFans\Declarations.vb" />
     <Compile Include="API\JustForFans\M3U8.vb" />