From f0014d287489670cd8ebdf6665dc3e5bf6cc005b Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Sat, 19 Aug 2023 18:42:01 +0300 Subject: [PATCH] 2023.08.19.0 API.Base: update 'TokenBatch', 'GDLBatch'; add 'YTDLPBatch' API.Base.UserDataBase: update 'EraseData' function API.Redgifs: fix hd/sd issue API.TikTok: update dates in commands; replace 'TokenBatch' with 'YTDLPBatch' UserDownloadQueueForm: update functions --- SCrawler/API/Base/{GDLBatch.vb => GDL.vb} | 10 ------ SCrawler/API/Base/TokenBatch.vb | 10 ++++++ SCrawler/API/Base/UserDataBase.vb | 12 +++---- SCrawler/API/Base/YTDLP.vb | 18 ++++++++++ SCrawler/API/Redgifs/UserData.vb | 2 +- SCrawler/API/TikTok/UserData.vb | 17 +++++++-- SCrawler/Download/UserDownloadQueueForm.vb | 40 ++++++++++++---------- SCrawler/My Project/AssemblyInfo.vb | 4 +-- SCrawler/SCrawler.vbproj | 3 +- 9 files changed, 76 insertions(+), 40 deletions(-) rename SCrawler/API/Base/{GDLBatch.vb => GDL.vb} (85%) create mode 100644 SCrawler/API/Base/YTDLP.vb diff --git a/SCrawler/API/Base/GDLBatch.vb b/SCrawler/API/Base/GDL.vb similarity index 85% rename from SCrawler/API/Base/GDLBatch.vb rename to SCrawler/API/Base/GDL.vb index 9b19f4a..199f17c 100644 --- a/SCrawler/API/Base/GDLBatch.vb +++ b/SCrawler/API/Base/GDL.vb @@ -67,7 +67,6 @@ Namespace API.Base.GDL End Function End Module Friend Class GDLBatch : Inherits TokenBatch - Friend Property TempPostsList As List(Of String) Friend Const UrlLibStart As String = "[urllib3.connectionpool][debug]" Friend Const UrlTextStart As String = UrlLibStart & " https" Friend Sub New(ByVal _Token As Threading.CancellationToken) @@ -75,20 +74,11 @@ Namespace API.Base.GDL MainProcessName = "gallery-dl" ChangeDirectory(Settings.GalleryDLFile.File) End Sub - Public Overrides Sub Create() - If TempPostsList Is Nothing Then TempPostsList = New List(Of String) - MyBase.Create() - End Sub Protected Overrides Async Sub OutputDataReceiver(ByVal Sender As Object, ByVal e As DataReceivedEventArgs) If Not ProcessKilled Then MyBase.OutputDataReceiver(Sender, e) Await Validate(e.Data) End If End Sub - Protected Overridable Async Function Validate(ByVal Value As String) As Task - If Not ProcessKilled AndAlso Await Task.Run(Of Boolean)(Function() Token.IsCancellationRequested OrElse - (Not Value.IsEmptyString AndAlso - TempPostsList.Exists(Function(v) Value.Contains(v)))) Then Kill() - End Function End Class End Namespace \ No newline at end of file diff --git a/SCrawler/API/Base/TokenBatch.vb b/SCrawler/API/Base/TokenBatch.vb index 924c3a0..80133a7 100644 --- a/SCrawler/API/Base/TokenBatch.vb +++ b/SCrawler/API/Base/TokenBatch.vb @@ -10,11 +10,16 @@ Imports System.Threading Imports PersonalUtilities.Tools Namespace API.Base Friend Class TokenBatch : Inherits BatchExecutor + Friend Property TempPostsList As List(Of String) Protected ReadOnly Token As CancellationToken Friend Sub New(ByVal _Token As CancellationToken) MyBase.New(True) Token = _Token End Sub + Public Overrides Sub Create() + If TempPostsList Is Nothing Then TempPostsList = New List(Of String) + MyBase.Create() + End Sub Protected Overrides Async Sub OutputDataReceiver(ByVal Sender As Object, ByVal e As DataReceivedEventArgs) MyBase.OutputDataReceiver(Sender, e) Await Task.Run(Sub() If Token.IsCancellationRequested Then Kill()) @@ -23,5 +28,10 @@ Namespace API.Base MyBase.ErrorDataReceiver(Sender, e) Await Task.Run(Sub() If Token.IsCancellationRequested Then Kill()) End Sub + Protected Overridable Async Function Validate(ByVal Value As String) As Task + If Not ProcessKilled AndAlso Await Task.Run(Of Boolean)(Function() Token.IsCancellationRequested OrElse + (Not Value.IsEmptyString AndAlso + TempPostsList.Exists(Function(v) Value.Contains(v)))) Then Kill() + End Function End Class End Namespace \ No newline at end of file diff --git a/SCrawler/API/Base/UserDataBase.vb b/SCrawler/API/Base/UserDataBase.vb index 6e36220..b425cca 100644 --- a/SCrawler/API/Base/UserDataBase.vb +++ b/SCrawler/API/Base/UserDataBase.vb @@ -1829,12 +1829,6 @@ BlockNullPicture: If m.Contains(IUserData.EraseMode.History) Then If MyFilePosts.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True If MyFileData.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True - If result Then - _TempPostsList.Clear() - _TempMediaList.Clear() - _ContentNew.Clear() - _ContentList.Clear() - End If End If If m.Contains(IUserData.EraseMode.Data) Then Dim files As List(Of SFile) = SFile.GetFiles(DownloadContentDefault_GetRootDir.CSFileP,, SearchOption.AllDirectories, e) @@ -1843,6 +1837,12 @@ BlockNullPicture: LatestData.Clear() result = True End If + If result Then + _TempPostsList.Clear() + _TempMediaList.Clear() + _ContentNew.Clear() + _ContentList.Clear() + End If End If End If Return result diff --git a/SCrawler/API/Base/YTDLP.vb b/SCrawler/API/Base/YTDLP.vb new file mode 100644 index 0000000..0369bbf --- /dev/null +++ b/SCrawler/API/Base/YTDLP.vb @@ -0,0 +1,18 @@ +' Copyright (C) 2023 Andy https://github.com/AAndyProgram +' This program is free software: you can redistribute it and/or modify +' it under the terms of the GNU General Public License as published by +' the Free Software Foundation, either version 3 of the License, or +' (at your option) any later version. +' +' This program is distributed in the hope that it will be useful, +' but WITHOUT ANY WARRANTY +Namespace API.Base.YTDLP + Friend Class YTDLPBatch : Inherits GDL.GDLBatch + Friend Sub New(ByVal _Token As Threading.CancellationToken) + MyBase.New(_Token) + Commands.Clear() + MainProcessName = "yt-dlp" + ChangeDirectory(Settings.YtdlpFile.File) + End Sub + End Class +End Namespace \ No newline at end of file diff --git a/SCrawler/API/Redgifs/UserData.vb b/SCrawler/API/Redgifs/UserData.vb index 9a54290..54a312f 100644 --- a/SCrawler/API/Redgifs/UserData.vb +++ b/SCrawler/API/Redgifs/UserData.vb @@ -83,7 +83,7 @@ Namespace API.RedGifs If Not j Is Nothing Then With j("urls") If .ListExists Then - Dim u$ = If(.Item("hd"), .Item("sd")).XmlIfNothingValue + Dim u$ = .Value("hd").IfNullOrEmpty(.Value("sd")) If Not u.IsEmptyString Then Dim ut As UTypes = UTypes.Undefined 'Type 1: video diff --git a/SCrawler/API/TikTok/UserData.vb b/SCrawler/API/TikTok/UserData.vb index b6425d1..ec95a76 100644 --- a/SCrawler/API/TikTok/UserData.vb +++ b/SCrawler/API/TikTok/UserData.vb @@ -71,6 +71,7 @@ Namespace API.TikTok TitleUseNative = .Value(Name_TitleUseNative).FromXML(Of Boolean)(True) TitleAddVideoID = .Value(Name_TitleAddVideoID).FromXML(Of Boolean)(True) LastDownloadDate = AConvert(Of Date)(.Value(Name_LastDownloadDate), ADateTime.Formats.BaseDateTime, Nothing) + If Not LastDownloadDate.HasValue Then LastDownloadDate = LastUpdated Else .Add(Name_RemoveTagsFromTitle, RemoveTagsFromTitle.BoolToInteger) .Add(Name_TitleUseNative, TitleUseNative.BoolToInteger) @@ -109,12 +110,20 @@ Namespace API.TikTok End With If LastDownloadDate.HasValue Then - If dateAfter.HasValue And Not DownloadDateFrom.HasValue Then + If Not DownloadDateTo.HasValue And Not DownloadDateFrom.HasValue Then + If LastDownloadDate.Value.AddDays(1) <= Now Then + dateAfter = LastDownloadDate.Value + Else + dateAfter = LastDownloadDate.Value.AddDays(-1) + End If + dateBefore = Nothing + ElseIf dateAfter.HasValue And Not DownloadDateFrom.HasValue Then If (LastDownloadDate.Value - dateAfter.Value).TotalDays > 1 Then dateAfter = dateAfter.Value.AddDays(1) End If End If - Using b As New TokenBatch(Token) + Using b As New YTDLP.YTDLPBatch(Token) With {.TempPostsList = _TempPostsList} + b.Commands.Clear() b.ChangeDirectory(cache) b.Encoding = BatchExecutor.UnicodeEncoding b.Execute(CreateYTCommand(cache.RootDirectory, URL, False, dateBefore, dateAfter)) @@ -163,6 +172,7 @@ Namespace API.TikTok End If Next End If + If _TempMediaList.Count > 0 Then LastDownloadDate = Now Catch ex As Exception ProcessException(ex, Token, $"data downloading error [{URL}]") End Try @@ -210,6 +220,9 @@ Namespace API.TikTok command &= "-o %(id)s" End If End If + '#If DEBUG Then + 'Debug.WriteLine(command) + '#End If Return command End Function #End Region diff --git a/SCrawler/Download/UserDownloadQueueForm.vb b/SCrawler/Download/UserDownloadQueueForm.vb index 2dd9db7..639c841 100644 --- a/SCrawler/Download/UserDownloadQueueForm.vb +++ b/SCrawler/Download/UserDownloadQueueForm.vb @@ -77,10 +77,13 @@ Namespace DownloadObjects If b Then e.Handled = True End Sub Friend Sub Downloader_Downloading(ByVal Value As Boolean) - If Not Value Then ControlInvokeFast(LIST_QUEUE, Sub() - LIST_QUEUE.Items.Clear() - Tokens.ListClearDispose - End Sub, EDP.None) + Try + If Not Value Then ControlInvokeFast(LIST_QUEUE, Sub() + LIST_QUEUE.Items.Clear() + Tokens.ListClearDispose + End Sub, EDP.None) + Catch + End Try End Sub Friend Sub Downloader_UserDownloadStateChanged(ByVal User As IUserData, ByVal IsDownloading As Boolean) Try @@ -93,8 +96,8 @@ Namespace DownloadObjects LIST_QUEUE.Items.Remove(u) End If LIST_QUEUE.Refresh() - End Sub) - Catch ex As Exception + End Sub, EDP.None) + Catch End Try End Sub Private Sub User_UserDownloadStateChanged(ByVal User As IUserData, ByVal IsDownloading As Boolean) @@ -111,7 +114,7 @@ Namespace DownloadObjects LIST_QUEUE.Refresh() End If End If - End Sub) + End Sub, EDP.None) Catch End Try End Sub @@ -143,22 +146,23 @@ Namespace DownloadObjects Tokens.Add(token) End If End If - Catch ex As Exception + Catch End Try End Sub Private Sub FindUser() - Try - MainFrameObj.FocusUser(GetUserSelectedUser().Key, True) - Catch ex As Exception - End Try + Try : MainFrameObj.FocusUser(GetUserSelectedUser().Key, True) : Catch : End Try End Sub Private Function GetUserSelectedUser() As ListUser - Dim lu As ListUser = Nothing - ControlInvokeFast(LIST_QUEUE, Sub() - Dim sIndx% = LIST_QUEUE.SelectedIndex - If sIndx >= 0 Then lu = LIST_QUEUE.Items(sIndx) - End Sub) - Return lu + Try + Dim lu As ListUser = Nothing + ControlInvokeFast(LIST_QUEUE, Sub() + Dim sIndx% = LIST_QUEUE.SelectedIndex + If sIndx >= 0 Then lu = LIST_QUEUE.Items(sIndx) + End Sub, EDP.None) + Return lu + Catch + Return Nothing + End Try End Function End Class End Namespace \ No newline at end of file diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index dcb6375..4c93362 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/SCrawler.vbproj b/SCrawler/SCrawler.vbproj index 9fe8bb8..b96a3e0 100644 --- a/SCrawler/SCrawler.vbproj +++ b/SCrawler/SCrawler.vbproj @@ -168,13 +168,14 @@ - + +