diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb index 5de176c..f111dbe 100644 --- a/SCrawler/API/Instagram/UserData.vb +++ b/SCrawler/API/Instagram/UserData.vb @@ -973,7 +973,8 @@ Namespace API.Instagram Try Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/highlights/{ID}/highlights_tray/",, EDP.ThrowException) If Not r.IsEmptyString Then - Using j As EContainer = JsonDocument.Parse(r).XmlIfNothing()("tray").XmlIfNothing + Dim ee As New ErrorsDescriber(EDP.ReturnValue) With {.DeclaredMessage = New MMessage($"{ToStringForLog()}:")} + Using j As EContainer = JsonDocument.Parse(r, ee).XmlIfNothing()("tray").XmlIfNothing If j.Count > 0 Then Return j.Select(Function(jj) jj.Value("id").Replace("highlight:", String.Empty)).ListIfNothing End Using End If diff --git a/SCrawler/API/OnlyFans/UserData.vb b/SCrawler/API/OnlyFans/UserData.vb index 50f52c7..e5a93a2 100644 --- a/SCrawler/API/OnlyFans/UserData.vb +++ b/SCrawler/API/OnlyFans/UserData.vb @@ -444,42 +444,56 @@ Namespace API.OnlyFans Return f End Get End Property - Private Function UpdateSignature(ByVal Path As String, Optional ByVal ForceUpdateAuth As Boolean = False) As Boolean + Private Function UpdateSignature(ByVal Path As String, Optional ByVal ForceUpdateAuth As Boolean = False, + Optional ByVal Round As Integer = 0) As Boolean Try If UpdateAuthFile(ForceUpdateAuth) Then Const nullMsg$ = "The auth parameter is null" - Dim j As EContainer = JsonDocument.Parse(AuthFile.GetText) - Dim pattern$ = j.Value("format") - If pattern.IsEmptyString Then Throw New ArgumentNullException("format", nullMsg) - pattern = pattern.Replace("{}", "{0}").Replace("{:x}", "{1:x}") + Dim j As EContainer + Try + j = JsonDocument.Parse(AuthFile.GetText) + Catch jex As Exception + If Round = 0 Then + AuthFile.Delete() + UpdateAuthFile(True) + Return UpdateSignature(Path, ForceUpdateAuth, Round + 1) + Else + MySettings.SessionAborted = True + Return False + End If + End Try + If Not j Is Nothing Then + Dim pattern$ = j.Value("format") + If pattern.IsEmptyString Then Throw New ArgumentNullException("format", nullMsg) + pattern = pattern.Replace("{}", "{0}").Replace("{:x}", "{1:x}") - Dim li%() = j("checksum_indexes").Select(Function(e) CInt(e(0).Value)).ToArray + Dim li%() = j("checksum_indexes").Select(Function(e) CInt(e(0).Value)).ToArray - If Not li.ListExists Then Throw New ArgumentNullException("checksum_indexes", nullMsg) - If j.Value("static_param").IsEmptyString Then Throw New ArgumentNullException("static_param", nullMsg) - If j.Value("checksum_constant").IsEmptyString Then Throw New ArgumentNullException("checksum_constant", nullMsg) + If Not li.ListExists Then Throw New ArgumentNullException("checksum_indexes", nullMsg) + If j.Value("static_param").IsEmptyString Then Throw New ArgumentNullException("static_param", nullMsg) + If j.Value("checksum_constant").IsEmptyString Then Throw New ArgumentNullException("checksum_constant", nullMsg) - Dim t$ = ADateTime.ConvertToUnix64(Now.ToUniversalTime).ToString - Dim h$ = String.Join(vbLf, j.Value("static_param"), t, Path, MySettings.HH_USER_ID.Value.ToString) + Dim t$ = ADateTime.ConvertToUnix64(Now.ToUniversalTime).ToString + Dim h$ = String.Join(vbLf, j.Value("static_param"), t, Path, MySettings.HH_USER_ID.Value.ToString) - Dim hash$ = GetHashSha1(h) - Dim hashBytes() As Byte = System.Text.Encoding.ASCII.GetBytes(hash) - Dim hashSum% = li.Sum(Function(i) hashBytes(i)) + CInt(j.Value("checksum_constant")) - Dim sign$ = String.Format(pattern, hash, Math.Abs(hashSum)) + Dim hash$ = GetHashSha1(h) + Dim hashBytes() As Byte = System.Text.Encoding.ASCII.GetBytes(hash) + Dim hashSum% = li.Sum(Function(i) hashBytes(i)) + CInt(j.Value("checksum_constant")) + Dim sign$ = String.Format(pattern, hash, Math.Abs(hashSum)) - '#If DEBUG Then - 'Debug.WriteLine(sign) - 'Debug.WriteLine(t) - '#End If + '#If DEBUG Then + 'Debug.WriteLine(sign) + 'Debug.WriteLine(t) + '#End If - Responser.Headers.Add(HeaderSign, sign) - Responser.Headers.Add(HeaderTime, t) + Responser.Headers.Add(HeaderSign, sign) + Responser.Headers.Add(HeaderTime, t) - j.Dispose() - Return True - Else - Return False + j.Dispose() + Return True + End If End If + Return False Catch ex As Exception Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, $"{ToStringForLog()}: UpdateSignature", False) End Try @@ -526,7 +540,7 @@ Namespace API.OnlyFans Private _DownloadingException_AuthFileUpdate As Boolean = False Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False, Optional ByVal EObj As Object = Nothing) As Integer - If Responser.StatusCode = Net.HttpStatusCode.BadRequest Then + If Responser.StatusCode = Net.HttpStatusCode.BadRequest Then '400 If Not _DownloadingException_AuthFileUpdate AndAlso UpdateAuthFile(True) Then _DownloadingException_AuthFileUpdate = True Return 2 @@ -535,13 +549,17 @@ Namespace API.OnlyFans MyMainLOG = $"{ToStringForLog()}: OnlyFans credentials expired" Return 1 End If - ElseIf Responser.StatusCode = Net.HttpStatusCode.NotFound Then + ElseIf Responser.StatusCode = Net.HttpStatusCode.NotFound Then '404 UserExists = False Return 1 - ElseIf Responser.StatusCode = Net.HttpStatusCode.GatewayTimeout Or Responser.StatusCode = 429 Then + ElseIf Responser.StatusCode = Net.HttpStatusCode.GatewayTimeout Or Responser.StatusCode = 429 Then '504, 429 If Responser.StatusCode = 429 Then MyMainLOG = $"[429] OnlyFans too many requests ({ToStringForLog()})" MySettings.SessionAborted = True Return 1 + ElseIf Responser.StatusCode = Net.HttpStatusCode.Unauthorized Then '401 + MySettings.SessionAborted = True + MyMainLOG = $"{ToStringForLog()}: OnlyFans credentials expired" + Return 1 Else Return 0 End If diff --git a/SCrawler/API/Xhamster/UserData.vb b/SCrawler/API/Xhamster/UserData.vb index fae6378..0987d03 100644 --- a/SCrawler/API/Xhamster/UserData.vb +++ b/SCrawler/API/Xhamster/UserData.vb @@ -582,7 +582,8 @@ Namespace API.Xhamster #Region "Exception" Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False, Optional ByVal EObj As Object = Nothing) As Integer - Return If(Responser.Status = Net.WebExceptionStatus.ConnectionClosed, 1, 0) + '8, 503 + Return If(Responser.Status = Net.WebExceptionStatus.ConnectionClosed Or Responser.StatusCode = Net.HttpStatusCode.ServiceUnavailable, 1, 0) End Function #End Region #Region "IDisposable support" diff --git a/SCrawler/Download/Automation/Scheduler.vb b/SCrawler/Download/Automation/Scheduler.vb index 5bd9b81..ed8dc45 100644 --- a/SCrawler/Download/Automation/Scheduler.vb +++ b/SCrawler/Download/Automation/Scheduler.vb @@ -130,16 +130,25 @@ Namespace DownloadObjects #End Region #Region "Execution" Friend Async Function Start(ByVal Init As Boolean) As Task - Await Task.Run(Sub() - If Count > 0 Then - If Plans.Exists(PlanDownloading) Then PlansWaiter(PlanDownloading) - For Each Plan In Plans - Plan.Start(Init) - PlansWaiter(PlanDownloading) - Thread.Sleep(1000) - Next - End If - End Sub) + Try + Await Task.Run(Sub() + If Count > 0 Then + If Plans.Exists(PlanDownloading) Then PlansWaiter(PlanDownloading) + For Each Plan In Plans + Plan.Start(Init) + PlansWaiter(PlanDownloading) + Thread.Sleep(1000) + Next + End If + End Sub) + Catch ex As Exception + If Init Then + ErrorsDescriber.Execute(EDP.SendToLog, ex, "Start automation") + MainFrameObj.UpdateLogButton() + Else + Throw ex + End If + End Try End Function Friend Sub [Stop]() If Count > 0 Then Plans.ForEach(Sub(p) p.Stop()) diff --git a/SCrawler/Download/TDownloader.vb b/SCrawler/Download/TDownloader.vb index 65ae841..51141b7 100644 --- a/SCrawler/Download/TDownloader.vb +++ b/SCrawler/Download/TDownloader.vb @@ -156,9 +156,10 @@ Namespace DownloadObjects Private ReadOnly NProv As IFormatProvider #End Region #Region "Working, Count" - Friend ReadOnly Property Working As Boolean + Friend ReadOnly Property Working(Optional ByVal CheckThread As Boolean = True) As Boolean Get - Return _PoolReconfiguration Or (Pool.Count > 0 AndAlso Pool.Exists(Function(j) j.Working)) Or If(CheckerThread?.IsAlive, False) + Return _PoolReconfiguration Or (Pool.Count > 0 AndAlso Pool.Exists(Function(j) j.Working)) Or + (CheckThread AndAlso If(CheckerThread?.IsAlive, False)) End Get End Property Friend ReadOnly Property Count As Integer diff --git a/SCrawler/MainFrame.vb b/SCrawler/MainFrame.vb index 7383a32..6b33d26 100644 --- a/SCrawler/MainFrame.vb +++ b/SCrawler/MainFrame.vb @@ -578,10 +578,15 @@ CloseResume: ControlInvokeFast(Me, Sub() BTT_TRAY_PAUSE_AUTOMATION.Visible = b) End Sub Private Async Sub BTT_DOWN_AUTOMATION_Click(sender As Object, e As EventArgs) Handles BTT_DOWN_AUTOMATION.Click - Using f As New SchedulerEditorForm : f.ShowDialog() : End Using - Await Settings.Automation.Start(False) - UpdatePauseButtonsVisibility() - MainFrameObj.PauseButtons.UpdatePauseButtons() + Try + Using f As New SchedulerEditorForm : f.ShowDialog() : End Using + Await Settings.Automation.Start(False) + UpdatePauseButtonsVisibility() + MainFrameObj.PauseButtons.UpdatePauseButtons() + Catch ex As Exception + ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Start automation") + MainFrameObj.UpdateLogButton() + End Try End Sub Private Sub BTT_DOWN_AUTOMATION_PAUSE_Click(sender As Object, e As EventArgs) Handles BTT_DOWN_AUTOMATION_PAUSE.Click, BTT_TRAY_PAUSE_AUTOMATION.Click Dim p As PauseModes = Settings.Automation.Pause @@ -1981,7 +1986,7 @@ ResumeDownloadingOperation: MainFrameObj.UpdateLogButton() End Sub Private Sub Downloader_Downloading(ByVal Value As Boolean) - Dim __isDownloading As Boolean = Value Or Downloader.Working + Dim __isDownloading As Boolean = Value Or Downloader.Working(False) ControlInvokeFast(Toolbar_TOP, BTT_DOWN_STOP, Sub() BTT_DOWN_STOP.Enabled = __isDownloading) TrayIcon.Icon = If(__isDownloading, My.Resources.ArrowDownIcon_Blue_24, My.Resources.RainbowIcon_48) End Sub