From f8116fd048c02e07debf9b19c17fc1563452a064 Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Mon, 18 Sep 2023 07:53:29 +0300 Subject: [PATCH] 2023.9.18.0 API.Instagram: handle error 500; fix saved posts bug API.Reddit: disable token refresh if there are no profiles to download API.UserDataBind: consolidate colors; update labels only for the added user AutoDownloader: change pause event; update pause function for scheduler; fix incorrect pause in scheduler; add icon for SchedulerEditorForm --- SCrawler/API/Instagram/UserData.vb | 18 +++---- SCrawler/API/Reddit/SiteSettings.vb | 42 +++++++++++++++- SCrawler/API/UserDataBind.vb | 17 +++++-- .../Download/Automation/AutoDownloader.vb | 5 +- .../Automation/AutoDownloaderPauseButtons.vb | 49 ++++++++++--------- SCrawler/Download/Automation/Scheduler.vb | 19 ++++--- .../SchedulerEditorForm.Designer.vb | 2 +- .../Automation/SchedulerEditorForm.vb | 6 ++- SCrawler/GlobalSuppressions.vb | 3 +- SCrawler/MainFrame.vb | 2 +- 10 files changed, 110 insertions(+), 53 deletions(-) diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb index 3a94598..47026d0 100644 --- a/SCrawler/API/Instagram/UserData.vb +++ b/SCrawler/API/Instagram/UserData.vb @@ -637,10 +637,8 @@ Namespace API.Instagram NextCursor = .Value("next_max_id") If .Contains("items") Then nodes = (From ee As EContainer In .Item("items") Where ee.Count > 0 Select ee(0)) End With - If nodes.ListExists Then - DefaultParser(nodes, Sections.SavedPosts, Token) - If HasNextPage And Not NextCursor.IsEmptyString Then SavedPostsDownload(NextCursor, Token) - End If + If nodes.ListExists AndAlso DefaultParser(nodes, Sections.SavedPosts, Token) AndAlso + HasNextPage AndAlso Not NextCursor.IsEmptyString Then SavedPostsDownload(NextCursor, Token) End If End Using End If @@ -954,15 +952,15 @@ Namespace API.Instagram ''' Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False, Optional ByVal s As Object = Nothing) As Integer - If Responser.StatusCode = HttpStatusCode.NotFound Then + If Responser.StatusCode = HttpStatusCode.NotFound Then '404 If Not UserNameRequested AndAlso GetUserNameById() Then Return 1 Else UserExists = False - ElseIf Responser.StatusCode = HttpStatusCode.BadRequest Then + ElseIf Responser.StatusCode = HttpStatusCode.BadRequest Then '400 HasError = True MyMainLOG = $"Instagram credentials have expired [{CInt(Responser.StatusCode)}]: {ToStringForLog()} [{s}]" DisableSection(s) - ElseIf Responser.StatusCode = HttpStatusCode.Forbidden And s = Sections.Tagged Then + ElseIf Responser.StatusCode = HttpStatusCode.Forbidden And s = Sections.Tagged Then '403 Return 3 - ElseIf Responser.StatusCode = 429 Then + ElseIf Responser.StatusCode = 429 Then '429 With MySiteSettings Dim WaiterExists As Boolean = .LastApplyingValue.HasValue .TooManyRequests(True) @@ -971,10 +969,10 @@ Namespace API.Instagram Caught429 = True MyMainLOG = $"Number of requests before error 429: {RequestsCount}" Return 1 - ElseIf Responser.StatusCode = 560 Then + ElseIf Responser.StatusCode = 560 Or Responser.StatusCode = HttpStatusCode.InternalServerError Then '560, 500 MySiteSettings.SkipUntilNextSession = True Else - MyMainLOG = $"Something is wrong. Your credentials may have expired [{CInt(Responser.StatusCode)}]: {ToString()} [{s}]" + MyMainLOG = $"Something is wrong. Your credentials may have expired [{CInt(Responser.StatusCode)}/{CInt(Responser.Status)}]: {ToString()} [{s}]" DisableSection(s) If Not FromPE Then LogError(ex, Message) : HasError = True Return 0 diff --git a/SCrawler/API/Reddit/SiteSettings.vb b/SCrawler/API/Reddit/SiteSettings.vb index 86c7d73..7031469 100644 --- a/SCrawler/API/Reddit/SiteSettings.vb +++ b/SCrawler/API/Reddit/SiteSettings.vb @@ -104,12 +104,45 @@ Namespace API.Reddit Return New UserData End Function #End Region -#Region "Available, UpdateRedGifsToken" +#Region "DownloadStarted, ReadyToDownload, Available, DownloadDone, UpdateRedGifsToken" + Private ____DownloadStarted As Boolean = False + Friend Overrides Sub DownloadStarted(ByVal What As Download) + If What = Download.Main Then ____DownloadStarted = True + MyBase.DownloadStarted(What) + End Sub Friend Property SessionInterrupted As Boolean = False Friend Overrides Function ReadyToDownload(ByVal What As Download) As Boolean - If What = Download.Main Then Return Not SessionInterrupted Else Return True + If What = Download.Main Then + Dim result As Boolean = Not SessionInterrupted + If result Then + If ____DownloadStarted And ____AvailableRequested Then + ____AvailableResult = AvailableImpl(What, ____AvailableSilent) + ____AvailableChecked = True + ____AvailableRequested = False + result = ____AvailableResult + ElseIf ____AvailableChecked Then + result = ____AvailableResult + End If + End If + Return result + Else + Return True + End If End Function + Private ____AvailableRequested As Boolean = False + Private ____AvailableSilent As Boolean = True + Private ____AvailableChecked As Boolean = False + Private ____AvailableResult As Boolean = False Friend Overrides Function Available(ByVal What As Download, ByVal Silent As Boolean) As Boolean + If What = Download.Main And ____DownloadStarted Then + ____AvailableRequested = True + ____AvailableSilent = Silent + Return True + Else + Return AvailableImpl(What, Silent) + End If + End Function + Private Function AvailableImpl(ByVal What As Download, ByVal Silent As Boolean) As Boolean Try Dim trueValue As Boolean = Not What = Download.SavedPosts OrElse (Responser.CookiesExists And ACheck(SavedPostsUserName.Value)) If Not trueValue Then Return False @@ -141,6 +174,11 @@ Namespace API.Reddit End Function Friend Overrides Sub DownloadDone(ByVal What As Download) SessionInterrupted = False + ____DownloadStarted = False + ____AvailableRequested = False + ____AvailableChecked = False + ____AvailableSilent = True + ____AvailableResult = False MyBase.DownloadDone(What) End Sub Private Sub UpdateRedGifsToken() diff --git a/SCrawler/API/UserDataBind.vb b/SCrawler/API/UserDataBind.vb index 9fbaf4a..c669b01 100644 --- a/SCrawler/API/UserDataBind.vb +++ b/SCrawler/API/UserDataBind.vb @@ -507,8 +507,9 @@ Namespace API .Temporary = Temporary .Favorite = Favorite .ReadyForDownload = ReadyForDownload - ConsolidateLabels() + ConsolidateLabels(_Item) ConsolidateScripts() + ConsolidateColors(_Item) .UpdateUserInformation() End If MainFrameObj.ImageHandler(_Item, False) @@ -546,12 +547,22 @@ Namespace API Catch ex As Exception End Try End Sub - Private Sub ConsolidateLabels() - UpdateLabels(Me, ListAddList(Nothing, Labels.ListWithRemove(SpecialLabels)), 1, True) + Private Sub ConsolidateLabels(ByVal Destination As UserDataBase) + UpdateLabels(If(Destination, Me), ListAddList(Nothing, Labels.ListWithRemove(SpecialLabels)), 1, True) End Sub Private Sub ConsolidateScripts() If Count > 1 AndAlso ScriptUse Then Collections.ForEach(Sub(c) c.ScriptUse = True) End Sub + Private Sub ConsolidateColors(ByVal Destination As UserDataBase) + If Count > 0 And Not Destination.ForeColor.HasValue And Not Destination.BackColor.HasValue Then + Dim b As Color? = BackColor + Dim f As Color? = ForeColor + If b.HasValue AndAlso Not Collections.All(Function(u) Not u Is Destination AndAlso u.BackColor.HasValue AndAlso u.BackColor.Value = b.Value) Then b = Nothing + If f.HasValue AndAlso Not Collections.All(Function(u) Not u Is Destination AndAlso u.ForeColor.HasValue AndAlso u.ForeColor.Value = f.Value) Then f = Nothing + If b.HasValue Then Destination.BackColor = b + If f.HasValue Then Destination.ForeColor = f + End If + End Sub #End Region #Region "Move, Merge" Friend Overrides Function MoveFiles(ByVal __CollectionName As String, ByVal __SpecialCollectionPath As SFile) As Boolean diff --git a/SCrawler/Download/Automation/AutoDownloader.vb b/SCrawler/Download/Automation/AutoDownloader.vb index b11bee2..e50bf34 100644 --- a/SCrawler/Download/Automation/AutoDownloader.vb +++ b/SCrawler/Download/Automation/AutoDownloader.vb @@ -15,7 +15,7 @@ Imports PersonalUtilities.Tools Imports PersonalUtilities.Tools.Notifications Namespace DownloadObjects Friend Class AutoDownloader : Inherits GroupParameters : Implements IIndexable, IEContainerProvider - Friend Event PauseDisabled() + Friend Event PauseChanged(ByVal Value As PauseModes) Friend Enum Modes As Integer None = 0 [Default] = 1 @@ -412,6 +412,7 @@ Namespace DownloadObjects Case PauseModes.Until : _PauseValue = DateLimit Case Else : _PauseValue = Nothing End Select + RaiseEvent PauseChanged(p) End Set End Property Private ReadOnly Property IsPaused As Boolean @@ -423,7 +424,7 @@ Namespace DownloadObjects Else _Pause = PauseModes.Disabled _PauseValue = Nothing - RaiseEvent PauseDisabled() + RaiseEvent PauseChanged(_Pause) Return False End If Else diff --git a/SCrawler/Download/Automation/AutoDownloaderPauseButtons.vb b/SCrawler/Download/Automation/AutoDownloaderPauseButtons.vb index be48b73..3510840 100644 --- a/SCrawler/Download/Automation/AutoDownloaderPauseButtons.vb +++ b/SCrawler/Download/Automation/AutoDownloaderPauseButtons.vb @@ -92,34 +92,39 @@ Namespace DownloadObjects BTT_PAUSE_H6.Click, BTT_PAUSE_H12.Click, BTT_PAUSE_UNTIL.Click, BTT_PAUSE_UNLIMITED.Click, BTT_PAUSE_DISABLE.Click - If (Place = ButtonsPlace.Scheduler And PlanIndex >= 0 And PlanIndex.ValueBetween(0, Settings.Automation.Count - 1)) Or Not Place = ButtonsPlace.Scheduler Then - Dim p As PauseModes = CInt(AConvert(Of Integer)(Sender.Tag, -10)) - If p > -10 Then - Dim d As Date? = Nothing - Dim _SetPauseValue As Action = Sub() - If Place = ButtonsPlace.Scheduler And PlanIndex.ValueBetween(0, Settings.Automation.Count - 1) Then - Settings.Automation(PlanIndex).Pause(d) = p - ElseIf Not Place = ButtonsPlace.Scheduler Then - Settings.Automation.Pause(d) = p - End If - End Sub - If p = PauseModes.Until Then - Using f As New DateTimeSelectionForm(TimeSelectionModes.End + TimeSelectionModes.Date + TimeSelectionModes.Time, Settings.Design) - f.ShowDialog() - If f.DialogResult = DialogResult.OK Then d = f.MyDateEnd - End Using - If d.HasValue Then _SetPauseValue.Invoke - Else - _SetPauseValue.Invoke - End If - UpdatePauseButtons() + Dim p As PauseModes = CInt(AConvert(Of Integer)(Sender.Tag, -10)) + If p > -10 AndAlso ((Place = ButtonsPlace.Scheduler And PlanIndex >= 0 And PlanIndex.ValueBetween(0, Settings.Automation.Count - 1)) OrElse + Not Place = ButtonsPlace.Scheduler OrElse + (Place = ButtonsPlace.Scheduler AndAlso PlanIndex = -1 AndAlso + MsgBoxE({$"Do you want to turn {IIf(p = PauseModes.Disabled, "off", "on")} pause for all plans?", "Pause plan"}, + vbExclamation + vbYesNo) = vbYes)) Then + Dim d As Date? = Nothing + Dim _SetPauseValue As Action = Sub() + If Place = ButtonsPlace.Scheduler And PlanIndex.ValueBetween(0, Settings.Automation.Count - 1) Then + Settings.Automation(PlanIndex).Pause(d) = p + ElseIf Not Place = ButtonsPlace.Scheduler Or Place = ButtonsPlace.Scheduler And PlanIndex = -1 Then + Settings.Automation.Pause(d) = p + End If + End Sub + If p = PauseModes.Until Then + Using f As New DateTimeSelectionForm(TimeSelectionModes.End + TimeSelectionModes.Date + TimeSelectionModes.Time, Settings.Design) + f.ShowDialog() + If f.DialogResult = DialogResult.OK Then d = f.MyDateEnd + End Using + If d.HasValue Then _SetPauseValue.Invoke + Else + _SetPauseValue.Invoke End If - ElseIf Place = ButtonsPlace.Scheduler And PlanIndex = -1 Then + UpdatePauseButtons() + ElseIf p > -10 And Place = ButtonsPlace.Scheduler And PlanIndex = -1 Then MsgBoxE({"The plan to be paused is not selected", "Pause plan"}, vbExclamation) End If End Sub #End Region #Region "Update buttons" + Friend Overloads Sub UpdatePauseButtons_Handler(ByVal Value As PauseModes) + UpdatePauseButtons() + End Sub Friend Overloads Sub UpdatePauseButtons() Handles TrayButtons.Updating UpdatePauseButtons(True) End Sub diff --git a/SCrawler/Download/Automation/Scheduler.vb b/SCrawler/Download/Automation/Scheduler.vb index a5426cc..5bd9b81 100644 --- a/SCrawler/Download/Automation/Scheduler.vb +++ b/SCrawler/Download/Automation/Scheduler.vb @@ -14,9 +14,9 @@ Imports PauseModes = SCrawler.DownloadObjects.AutoDownloader.PauseModes Namespace DownloadObjects Friend Class Scheduler : Implements IEnumerable(Of AutoDownloader), IMyEnumerator(Of AutoDownloader), IDisposable Friend Const Name_Plan As String = "Plan" - Friend Event PauseDisabled As AutoDownloader.PauseDisabledEventHandler - Private Sub OnPauseDisabled() - RaiseEvent PauseDisabled() + Friend Event PauseChanged As AutoDownloader.PauseChangedEventHandler + Private Sub OnPauseChanged(ByVal Value As PauseModes) + RaiseEvent PauseChanged(Pause) End Sub Private ReadOnly Plans As List(Of AutoDownloader) Friend Const FileNameDefault As String = "AutoDownload" @@ -31,7 +31,7 @@ Namespace DownloadObjects Plans = New List(Of AutoDownloader) File = Settings.AutomationFile.Value.IfNullOrEmpty(FileDefault) If Not File.Exists Then File = FileDefault - Reset(File) + Reset(File, True) End Sub Default Friend ReadOnly Property Item(ByVal Index As Integer) As AutoDownloader Implements IMyEnumerator(Of AutoDownloader).MyEnumeratorObject Get @@ -53,7 +53,7 @@ Namespace DownloadObjects End Function Friend Sub Add(ByVal Plan As AutoDownloader) Plan.Source = Me - AddHandler Plan.PauseDisabled, AddressOf OnPauseDisabled + AddHandler Plan.PauseChanged, AddressOf OnPauseChanged Plans.Add(Plan) Plans.ListReindex Update() @@ -87,16 +87,14 @@ Namespace DownloadObjects Catch End Try End Sub - Friend Function Reset(ByVal f As SFile) As Boolean - Dim __pause As PauseModes = Pause + Friend Function Reset(ByVal f As SFile, ByVal IsInit As Boolean) As Boolean If Plans.Count > 0 Then If Not Plans.Exists(PlanWorking) Then Pause = PauseModes.Unlimited If Plans.Exists(PlanWorking) Then MsgBoxE({$"Some plans are already being worked.{vbCr}Wait for the plans to complete their work and try again.", "Change scheduler"}, vbCritical) - If __pause = PauseModes.Until Then __pause = PauseModes.Unlimited - Pause = __pause + Pause = PauseModes.Unlimited Return False End If End If @@ -116,7 +114,8 @@ Namespace DownloadObjects End Using If Plans.Count > 0 Then Plans.ForEach(Sub(ByVal p As AutoDownloader) p.Source = Me - AddHandler p.PauseDisabled, AddressOf OnPauseDisabled + If Not IsInit Then p.Pause = PauseModes.Unlimited + AddHandler p.PauseChanged, AddressOf OnPauseChanged End Sub) : Plans.ListReindex End If Return True diff --git a/SCrawler/Download/Automation/SchedulerEditorForm.Designer.vb b/SCrawler/Download/Automation/SchedulerEditorForm.Designer.vb index 4aec812..f64c789 100644 --- a/SCrawler/Download/Automation/SchedulerEditorForm.Designer.vb +++ b/SCrawler/Download/Automation/SchedulerEditorForm.Designer.vb @@ -63,7 +63,7 @@ Namespace DownloadObjects Me.KeyPreview = True Me.MinimumSize = New System.Drawing.Size(430, 380) Me.Name = "SchedulerEditorForm" - Me.ShowIcon = False + Me.ShowIcon = True Me.ShowInTaskbar = False Me.Text = "Scheduler" CONTAINER_MAIN.ContentPanel.ResumeLayout(False) diff --git a/SCrawler/Download/Automation/SchedulerEditorForm.vb b/SCrawler/Download/Automation/SchedulerEditorForm.vb index 4edd83f..0ca8368 100644 --- a/SCrawler/Download/Automation/SchedulerEditorForm.vb +++ b/SCrawler/Download/Automation/SchedulerEditorForm.vb @@ -8,6 +8,7 @@ ' but WITHOUT ANY WARRANTY Imports PersonalUtilities.Forms Imports PersonalUtilities.Forms.Toolbars +Imports PersonalUtilities.Tools Imports ECI = PersonalUtilities.Forms.Toolbars.EditToolbar.ControlItem Imports ADB = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons Namespace DownloadObjects @@ -98,6 +99,7 @@ Namespace DownloadObjects } PauseArr = New AutoDownloaderPauseButtons(AutoDownloaderPauseButtons.ButtonsPlace.Scheduler) With { .MainFrameButtonsInstance = MainFrameObj.PauseButtons} + Icon = ImageRenderer.GetIcon(My.Resources.ScriptPic_32, EDP.ReturnValue) End Sub #End Region #Region "Form handlers" @@ -243,13 +245,14 @@ Namespace DownloadObjects Else f = $"{SettingsFolderName}\{Scheduler.FileNameDefault}_{selectedName.StringRemoveWinForbiddenSymbols}.xml" End If - If Not Settings.Automation.File = f AndAlso Settings.Automation.Reset(f) Then + If Not Settings.Automation.File = f AndAlso Settings.Automation.Reset(f, False) Then Settings.Automation.File = f If selectedName = defName Then Settings.AutomationFile.Value = Nothing Else Settings.AutomationFile.Value = f End If + PauseArr.UpdatePauseButtons() Refill() If Not .DataSource.Count = l.Count Then For i = l.Count - 1 To 0 Step -1 @@ -261,6 +264,7 @@ Namespace DownloadObjects End If End With End Using + l.Clear() Else MsgBoxE({"There are no plans created", msgTitle}, vbExclamation) End If diff --git a/SCrawler/GlobalSuppressions.vb b/SCrawler/GlobalSuppressions.vb index 760fd35..fae0414 100644 --- a/SCrawler/GlobalSuppressions.vb +++ b/SCrawler/GlobalSuppressions.vb @@ -8,4 +8,5 @@ Imports System.Diagnostics.CodeAnalysis - \ No newline at end of file + + diff --git a/SCrawler/MainFrame.vb b/SCrawler/MainFrame.vb index 61a24de..993bc8c 100644 --- a/SCrawler/MainFrame.vb +++ b/SCrawler/MainFrame.vb @@ -122,7 +122,7 @@ Public Class MainFrame .Automation = New Scheduler AddHandler .Groups.Updated, AddressOf .Automation.GROUPS_Updated AddHandler .Groups.Deleted, AddressOf .Automation.GROUPS_Deleted - AddHandler .Automation.PauseDisabled, AddressOf MainFrameObj.PauseButtons.UpdatePauseButtons + AddHandler .Automation.PauseChanged, AddressOf MainFrameObj.PauseButtons.UpdatePauseButtons_Handler If .Automation.Count > 0 Then .Labels.AddRange(.Automation.GetGroupsLabels, False) : .Labels.Update() _UFinit = False Await .Automation.Start(True)