From 46372ec9fbd373c653c20ff8aaed1dc052db6079 Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Sat, 13 Apr 2024 10:10:27 +0300 Subject: [PATCH] 2024.4.13.0 YT Add subtitles to information about downloaded files PluginProvider IPluginContentProvider: add 'ResetHistoryData' function SCrawler UserDataBase: call 'UpdateUsersList' when 'UpdateUserInformation' with argument 'OnlyDiff'; implement 'ResetHistoryData' function; set 'LastUpdated' to null and 'UpdateUserInformation' when erasing history data API.Instagram: set 'FirstLoadingDone' to false when erasing history data; fix broken saved posts downloading API.TikTok: set 'LastDownloadDate' to null when erasing history data API.YouTube: set last download dates to null when erasing history data GroupUsersViewer: add the number of users, object type and object name to the form title; add 'F1' to help hint; add '(Alt+)F3' to edit user AutoDownloaderEditorForm, SchedulerEditorForm, GroupEditorForm, GroupListForm: update to 'GroupUsersViewer' MainFrame: make 'EditSelectedUser' friend UserDataHost: implement 'ResetHistoryData' function SettingsCLS: add 'OnlyDiff' to the 'UpdateUsersList' function UserInfo: add 'ExactEquals' shared function --- Changelog.md | 14 ++++++++ .../Interfaces/IPluginContentProvider.vb | 1 + .../My Project/AssemblyInfo.vb | 4 +-- SCrawler.YouTube/My Project/AssemblyInfo.vb | 4 +-- .../Objects/YouTubeMediaContainerBase.vb | 22 +++++++------ .../My Project/AssemblyInfo.vb | 4 +-- SCrawler/API/Base/UserDataBase.vb | 6 ++-- SCrawler/API/Instagram/UserData.vb | 5 +-- SCrawler/API/TikTok/UserData.vb | 5 +++ SCrawler/API/YouTube/UserData.vb | 7 ++++ .../Automation/AutoDownloaderEditorForm.vb | 2 +- .../Automation/SchedulerEditorForm.vb | 2 +- SCrawler/Download/Groups/GroupEditorForm.vb | 2 +- SCrawler/Download/Groups/GroupListForm.vb | 8 +++-- SCrawler/Download/Groups/GroupUsersViewer.vb | 32 ++++++++++++++++--- SCrawler/MainFrame.vb | 4 +-- SCrawler/My Project/AssemblyInfo.vb | 4 +-- .../PluginsEnvironment/Hosts/UserDataHost.vb | 3 ++ SCrawler/SettingsCLS.vb | 11 +++++-- SCrawler/UserInfo.vb | 16 ++++++++++ 20 files changed, 118 insertions(+), 38 deletions(-) diff --git a/Changelog.md b/Changelog.md index 074d984..b5cc29b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,17 @@ +# 2024.4.13.0 + +*2024-04-13* + +- Added + - Minor improvements +- PluginProvider + - IPluginContentProvider: added `ResetHistoryData` function +- Fixed + - Sites + - TikTok: remove last download date when erasing history data + - YouTube: remove last download date when erasing history data + - Instagram: **saved posts aren't downloading** + # 2024.4.10.0 *2024-04-10* diff --git a/SCrawler.PluginProvider/Interfaces/IPluginContentProvider.vb b/SCrawler.PluginProvider/Interfaces/IPluginContentProvider.vb index a666861..fc82097 100644 --- a/SCrawler.PluginProvider/Interfaces/IPluginContentProvider.vb +++ b/SCrawler.PluginProvider/Interfaces/IPluginContentProvider.vb @@ -40,5 +40,6 @@ Namespace Plugin Sub GetMedia(ByVal Token As Threading.CancellationToken) Sub Download(ByVal Token As Threading.CancellationToken) Sub DownloadSingleObject(ByVal Data As IDownloadableMedia, ByVal Token As Threading.CancellationToken) + Sub ResetHistoryData() End Interface End Namespace \ No newline at end of file diff --git a/SCrawler.PluginProvider/My Project/AssemblyInfo.vb b/SCrawler.PluginProvider/My Project/AssemblyInfo.vb index 3eb453e..5c197ea 100644 --- a/SCrawler.PluginProvider/My Project/AssemblyInfo.vb +++ b/SCrawler.PluginProvider/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler.YouTube/My Project/AssemblyInfo.vb b/SCrawler.YouTube/My Project/AssemblyInfo.vb index d9f5c1f..6cfe06a 100644 --- a/SCrawler.YouTube/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTube/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb index aa98218..fee7244 100644 --- a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb +++ b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb @@ -1219,19 +1219,21 @@ Namespace API.YouTube.Objects 'Subtitles ThrowAny(Token) - If PostProcessing_OutputSubtitlesFormats.Count > 0 Then + If SubtitlesSelectedIndexes.Count > 0 And Not OutputSubtitlesFormat.IsEmptyString Then files = SFile.GetFiles(File, String.Format(fPatternFiles, OutputSubtitlesFormat.StringToLower),, EDP.ReturnValue) - AddFile(files) If files.ListExists Then - For Each f In files - For Each format In PostProcessing_OutputSubtitlesFormats - format = format.StringToLower - commandFile = $"{f.PathWithSeparator}{f.Name}.{format}" - AddFile(commandFile) - ThrowAny(Token) - .Execute($"ffmpeg -i ""{f}"" ""{commandFile}""") + AddFile(files) + If PostProcessing_OutputSubtitlesFormats.Count > 0 Then + For Each f In files + For Each format In PostProcessing_OutputSubtitlesFormats + format = format.StringToLower + commandFile = $"{f.PathWithSeparator}{f.Name}.{format}" + AddFile(commandFile) + ThrowAny(Token) + .Execute($"ffmpeg -i ""{f}"" ""{commandFile}""") + Next Next - Next + End If End If End If diff --git a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb index 44f76a0..71c317b 100644 --- a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/API/Base/UserDataBase.vb b/SCrawler/API/Base/UserDataBase.vb index 9d806e8..ee823b4 100644 --- a/SCrawler/API/Base/UserDataBase.vb +++ b/SCrawler/API/Base/UserDataBase.vb @@ -1001,7 +1001,7 @@ BlockNullPicture: x.Save(MyFileSettings) End Using - If Not IsSavedPosts Then Settings.UpdateUsersList(User) + If Not IsSavedPosts Then Settings.UpdateUsersList(User, True) Catch ex As Exception LogError(ex, "user information saving error") End Try @@ -1894,7 +1894,9 @@ BlockNullPicture: If m.Contains(IUserData.EraseMode.History) Then If MyFilePosts.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True If MyFileData.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True + LastUpdated = Nothing EraseData_AdditionalDataFiles() + UpdateUserInformation() End If If m.Contains(IUserData.EraseMode.Data) Then Dim files As List(Of SFile) = SFile.GetFiles(DownloadContentDefault_GetRootDir.CSFileP,, SearchOption.AllDirectories, e) @@ -1916,7 +1918,7 @@ BlockNullPicture: Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, $"EraseData({CInt(Mode)}): {ToStringForLog()}", False) End Try End Function - Protected Overridable Sub EraseData_AdditionalDataFiles() + Protected Overridable Sub EraseData_AdditionalDataFiles() Implements IPluginContentProvider.ResetHistoryData End Sub Friend Overridable Function Delete(Optional ByVal Multiple As Boolean = False, Optional ByVal CollectionValue As Integer = -1) As Integer Implements IUserData.Delete Dim f As SFile = SFile.GetPath(MyFile.CutPath.Path) diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb index 4429044..9d6500c 100644 --- a/SCrawler/API/Instagram/UserData.vb +++ b/SCrawler/API/Instagram/UserData.vb @@ -886,8 +886,8 @@ NextPageBlock: PostIDKV = New PostKV(.Value("code"), .Value("id"), Section) PostOriginUrl = DefaultParser_PostUrlCreator(PostIDKV) Pinned = .Contains("timeline_pinned_user_ids") - If (Section = Sections.Timeline And Not DefaultParser_IgnorePass) AndAlso PostKvExists(PostIDKV) Then - If Not Pinned Then Return False + If Not DefaultParser_IgnorePass AndAlso PostKvExists(PostIDKV) Then + If Not Section = Sections.Timeline OrElse Not Pinned Then Return False Else _TempPostsList.Add(PostIDKV.ID) PostsKVIDs.ListAddValue(PostIDKV, LNC) @@ -1233,6 +1233,7 @@ NextPageBlock: Protected Overrides Sub EraseData_AdditionalDataFiles() Dim f As SFile = MyFilePostsKV If f.Exists Then f.Delete(SFO.File, SFODelete.DeleteToRecycleBin, EDP.ReturnValue) + FirstLoadingDone = False End Sub #End Region #Region "Exceptions" diff --git a/SCrawler/API/TikTok/UserData.vb b/SCrawler/API/TikTok/UserData.vb index 21cd63e..a78d785 100644 --- a/SCrawler/API/TikTok/UserData.vb +++ b/SCrawler/API/TikTok/UserData.vb @@ -359,6 +359,11 @@ Namespace API.TikTok _TempMediaList.Add(m) End Sub #End Region +#Region "EraseData" + Protected Overrides Sub EraseData_AdditionalDataFiles() + LastDownloadDate = Nothing + End Sub +#End Region #Region "Exception" Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False, Optional ByVal EObj As Object = Nothing) As Integer diff --git a/SCrawler/API/YouTube/UserData.vb b/SCrawler/API/YouTube/UserData.vb index 4a49332..fab709d 100644 --- a/SCrawler/API/YouTube/UserData.vb +++ b/SCrawler/API/YouTube/UserData.vb @@ -467,6 +467,13 @@ Namespace API.YouTube _TempMediaList.Add(New UserMedia(Data)) End Sub #End Region +#Region "EraseData" + Protected Overrides Sub EraseData_AdditionalDataFiles() + LastDownloadDateVideos = Nothing + LastDownloadDateShorts = Nothing + LastDownloadDatePlaylist = Nothing + End Sub +#End Region #Region "DownloadingException" Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False, Optional ByVal EObj As Object = Nothing) As Integer diff --git a/SCrawler/Download/Automation/AutoDownloaderEditorForm.vb b/SCrawler/Download/Automation/AutoDownloaderEditorForm.vb index 9d08a31..eaebf45 100644 --- a/SCrawler/Download/Automation/AutoDownloaderEditorForm.vb +++ b/SCrawler/Download/Automation/AutoDownloaderEditorForm.vb @@ -100,7 +100,7 @@ Namespace DownloadObjects users.ListAddList(DownloadGroup.GetUsers(g)) End Using End If - GroupUsersViewer.Show(users) + GroupUsersViewer.Show(users, $"S {DEF_GROUP.TXT_NAME.Text}") users.Clear() End If Catch ex As Exception diff --git a/SCrawler/Download/Automation/SchedulerEditorForm.vb b/SCrawler/Download/Automation/SchedulerEditorForm.vb index a9d0ade..1bb045b 100644 --- a/SCrawler/Download/Automation/SchedulerEditorForm.vb +++ b/SCrawler/Download/Automation/SchedulerEditorForm.vb @@ -381,7 +381,7 @@ Namespace DownloadObjects users.ListAddList(Groups.DownloadGroup.GetUsers(.Self)) End If End If - Groups.GroupUsersViewer.Show(users) + Groups.GroupUsersViewer.Show(users, $"S { .Name}") users.Clear() End With End If diff --git a/SCrawler/Download/Groups/GroupEditorForm.vb b/SCrawler/Download/Groups/GroupEditorForm.vb index 8e43247..7912561 100644 --- a/SCrawler/Download/Groups/GroupEditorForm.vb +++ b/SCrawler/Download/Groups/GroupEditorForm.vb @@ -85,7 +85,7 @@ Namespace DownloadObjects.Groups If e = ShowUsersButtonKey Then Using g As New GroupParameters DEFS_GROUP.Get(g) - GroupUsersViewer.Show(DownloadGroup.GetUsers(g)) + GroupUsersViewer.Show(DownloadGroup.GetUsers(g), $"{IIf(FilterMode, "F", "G")} {g.Name}") End Using End If Catch ex As Exception diff --git a/SCrawler/Download/Groups/GroupListForm.vb b/SCrawler/Download/Groups/GroupListForm.vb index 9ccc5a0..23a5c70 100644 --- a/SCrawler/Download/Groups/GroupListForm.vb +++ b/SCrawler/Download/Groups/GroupListForm.vb @@ -305,10 +305,11 @@ Namespace DownloadObjects.Groups Try If _LatestSelected.ValueBetween(0, MyGroups.Count - 1) Then Dim i% + Dim n$ = String.Empty Dim users As New List(Of API.Base.IUserData) If Not IsViewFilter Then i = Settings.Groups.IndexOf(MyGroups(_LatestSelected)) - If i >= 0 Then users.ListAddList(DownloadGroup.GetUsers(Settings.Groups(i))) + If i >= 0 Then users.ListAddList(DownloadGroup.GetUsers(Settings.Groups(i))) : n = $"F {Settings.Groups(i).Name}" ElseIf _LatestSelected.ValueBetween(0, MyGroupParams.Count - 1) Then With MyGroupParams(_LatestSelected) If TypeOf .Self Is AutoDownloader Then @@ -325,14 +326,15 @@ Namespace DownloadObjects.Groups users.ListAddList(DownloadGroup.GetUsers(.Self)) End If End If + n = $"S { .Name}" End With ElseIf TypeOf .Self Is DownloadGroup Then i = Settings.Groups.IndexOf(.Name, .IsViewFilter) - If i >= 0 Then users.ListAddList(DownloadGroup.GetUsers(Settings.Groups(i))) + If i >= 0 Then users.ListAddList(DownloadGroup.GetUsers(Settings.Groups(i))) : n = $"G {Settings.Groups(i).Name}" End If End With End If - GroupUsersViewer.Show(users) + GroupUsersViewer.Show(users, n) users.Clear() End If Catch ex As Exception diff --git a/SCrawler/Download/Groups/GroupUsersViewer.vb b/SCrawler/Download/Groups/GroupUsersViewer.vb index f3b673e..13346e2 100644 --- a/SCrawler/Download/Groups/GroupUsersViewer.vb +++ b/SCrawler/Download/Groups/GroupUsersViewer.vb @@ -10,11 +10,16 @@ Imports SCrawler.API.Base Imports PersonalUtilities.Forms Namespace DownloadObjects.Groups Friend Class GroupUsersViewer : Inherits SimpleListForm(Of IUserData) - Friend Sub New(ByVal Users As IEnumerable(Of IUserData)) + Friend Sub New(ByVal Users As IEnumerable(Of IUserData), ByVal AdditText As String) MyBase.New(Users, Settings.Design) DesignXMLNodeName = "GroupUsersViewer" Provider = New CustomProvider(Function(u As UserDataBase) u.ToStringForLog) - FormText = "Users" + FormText = $"Users ({If(Users?.Count, 0)})" + If Not AdditText.IsEmptyString Then + Dim a$ = AdditText.Trim.Take(100).ListToStringE(String.Empty,,, " ", EDP.ReturnValue) + If Not a = AdditText Then AdditText &= "..." + FormText &= $" [{a}]" + End If Icon = My.Resources.UsersIcon_32 MyDefs.DelegateClosingChecker = False Mode = SimpleListFormModes.SelectedItems @@ -23,18 +28,26 @@ Namespace DownloadObjects.Groups End Sub Protected Overrides Sub MyForm_KeyDown(sender As Object, e As KeyEventArgs) Try + Dim b As Boolean = True If e.KeyCode = Keys.F And e.Control Then - e.Handled = True FocusUser() + ElseIf e.KeyCode = Keys.F3 Then + EditUser(e.Alt) ElseIf e = ShowUsersButtonKey Then MsgBoxE(New MMessage(DataSourceCollection.ListToStringE(vbCr, Provider,,, EDP.LogMessageValue), "User list") With {.Editable = True}) + ElseIf e.KeyCode = Keys.F1 And Not e.Alt And Not e.Control Then + MsgBoxE({$"Hotkeys:{vbCr}Alt+F1 - show user list{vbCr}Ctrl+F - find the selected user in the main window{vbCr}" & + $"F3 - edit selected user{vbCr}Alt+F3 - edit selected collection", "Hotkeys"}) + Else + b = False End If + If b Then e.Handled = True Catch End Try End Sub - Friend Overloads Shared Sub Show(ByVal Users As IEnumerable(Of IUserData)) + Friend Overloads Shared Sub Show(ByVal Users As IEnumerable(Of IUserData), ByVal AdditText As String) If Users.ListExists Then - MainFrameObj.OpenedGroupUsersForms.Add(New GroupUsersViewer(Users)) + MainFrameObj.OpenedGroupUsersForms.Add(New GroupUsersViewer(Users, AdditText)) MainFrameObj.OpenedGroupUsersForms.Last.Show() Else MsgBoxE({"No users were found based on the selected parameters", "Show group users"}, vbExclamation) @@ -49,6 +62,15 @@ Namespace DownloadObjects.Groups Catch End Try End Sub + Private Sub EditUser(ByVal EditCollection As Boolean) + Try + If _LatestSelected.ValueBetween(0, DataSourceCollection.Count - 1) Then + Dim u As IUserData = Settings.GetUser(DataSourceCollection(_LatestSelected).Key, EditCollection) + If Not u Is Nothing Then ControlInvokeFast(MainFrameObj.MF, Sub() MainFrameObj.MF.EditSelectedUser(u), EDP.None) + End If + Catch + End Try + End Sub Friend Overloads Sub Show() MyForm.Show() End Sub diff --git a/SCrawler/MainFrame.vb b/SCrawler/MainFrame.vb index 1e4873c..f9006e5 100644 --- a/SCrawler/MainFrame.vb +++ b/SCrawler/MainFrame.vb @@ -1706,9 +1706,9 @@ ResumeDownloadingOperation: End If End If End Sub - Private Sub EditSelectedUser() + Friend Sub EditSelectedUser(Optional ByVal CUser As IUserData = Nothing) Const MsgTitle$ = "User update" - Dim user As IUserData = GetSelectedUser() + Dim user As IUserData = If(CUser, GetSelectedUser()) If Not user Is Nothing Then On Error Resume Next If Not user.IsCollection OrElse DirectCast(user, UserDataBind).Count > 0 Then diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index dfcf27a..55df751 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/PluginsEnvironment/Hosts/UserDataHost.vb b/SCrawler/PluginsEnvironment/Hosts/UserDataHost.vb index 2400ffd..09a44cf 100644 --- a/SCrawler/PluginsEnvironment/Hosts/UserDataHost.vb +++ b/SCrawler/PluginsEnvironment/Hosts/UserDataHost.vb @@ -104,6 +104,9 @@ Namespace Plugin.Hosts End With End If End Sub + Protected Overrides Sub EraseData_AdditionalDataFiles() + ExternalPlugin.ResetHistoryData() + End Sub Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False, Optional ByVal EObj As Object = Nothing) As Integer LogError(ex, Message) diff --git a/SCrawler/SettingsCLS.vb b/SCrawler/SettingsCLS.vb index bc3c6e7..ed62480 100644 --- a/SCrawler/SettingsCLS.vb +++ b/SCrawler/SettingsCLS.vb @@ -825,14 +825,19 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable Return _UserListUpdateRequired End Get End Property - Friend Overloads Sub UpdateUsersList(ByVal u As UserInfo) + Friend Overloads Sub UpdateUsersList(ByVal u As UserInfo, Optional ByVal OnlyDiff As Boolean = False) + Dim result As Boolean = True Dim i% = UsersList.IndexOf(u) If i >= 0 Then - UsersList(i) = u + If Not OnlyDiff OrElse Not UserInfo.ExactEquals(UsersList(i), u) Then + UsersList(i) = u + Else + result = False + End If Else UsersList.Add(u) End If - UpdateUsersList() + If result Then UpdateUsersList() End Sub Friend Overloads Sub UpdateUsersList() Try diff --git a/SCrawler/UserInfo.vb b/SCrawler/UserInfo.vb index 71e34b6..1cc1401 100644 --- a/SCrawler/UserInfo.vb +++ b/SCrawler/UserInfo.vb @@ -182,6 +182,22 @@ Partial Friend Module MainMod End Function #End Region #Region "IEquatable Support" + Friend Shared Function ExactEquals(ByVal x As UserInfo, ByVal y As UserInfo) As Boolean + Return x.Name = y.Name And + x.Site = y.Site And + x.Plugin = y.Plugin And + x.AccountName = y.AccountName And + x.File = y.File And + x.IsSubscription = y.IsSubscription And + x.SpecialPath = y.SpecialPath And + x.SpecialCollectionPath = y.SpecialCollectionPath And + x.Merged = y.Merged And + x.UserModel = y.UserModel And + x.CollectionName = y.CollectionName And + x.CollectionModel = y.CollectionModel And + x.[Protected] = y.[Protected] And + AEquals(Of Date)(x.LastSeen, y.LastSeen) + End Function Friend Overloads Function Equals(ByVal Other As UserInfo) As Boolean Implements IEquatable(Of UserInfo).Equals Return Site.StringToLower = Other.Site.StringToLower And Name.StringToLower = Other.Name.StringToLower And IsSubscription = Other.IsSubscription And (Not Plugin = PathPlugin.PluginKey Or SpecialPath = Other.SpecialPath)