geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

2023.11.12.2

Browse Source 
API.Instagram: handle JSON primitive error (simple line)
API.OnlyFans: handle 401 error; handle file update error
API.Xhamster: handle 503 error
Scheduler: handle automation start error
This commit is contained in:
Andy
2023-11-12 22:03:14 +03:00
parent 5af0dcc46e
commit 95cbb6aeb1
6 changed files with 82 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
SCrawler/API/Instagram/UserData.vb
View File

@@ -973,7 +973,8 @@ Namespace API.Instagram
Try Try
Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/highlights/{ID}/highlights_tray/",, EDP.ThrowException) Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/highlights/{ID}/highlights_tray/",, EDP.ThrowException)
If Not r.IsEmptyString Then If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r).XmlIfNothing()("tray").XmlIfNothing Dim ee As New ErrorsDescriber(EDP.ReturnValue) With {.DeclaredMessage = New MMessage($"{ToStringForLog()}:")}
Using j As EContainer = JsonDocument.Parse(r, ee).XmlIfNothing()("tray").XmlIfNothing
If j.Count > 0 Then Return j.Select(Function(jj) jj.Value("id").Replace("highlight:", String.Empty)).ListIfNothing If j.Count > 0 Then Return j.Select(Function(jj) jj.Value("id").Replace("highlight:", String.Empty)).ListIfNothing
End Using End Using
End If End If

74
SCrawler/API/OnlyFans/UserData.vb
View File

@@ -444,42 +444,56 @@ Namespace API.OnlyFans
Return f Return f
End Get End Get
End Property End Property
Private Function UpdateSignature(ByVal Path As String, Optional ByVal ForceUpdateAuth As Boolean = False) As Boolean Private Function UpdateSignature(ByVal Path As String, Optional ByVal ForceUpdateAuth As Boolean = False,
Optional ByVal Round As Integer = 0) As Boolean
Try Try
If UpdateAuthFile(ForceUpdateAuth) Then If UpdateAuthFile(ForceUpdateAuth) Then
Const nullMsg$ = "The auth parameter is null" Const nullMsg$ = "The auth parameter is null"
Dim j As EContainer = JsonDocument.Parse(AuthFile.GetText) Dim j As EContainer
Dim pattern$ = j.Value("format") Try
If pattern.IsEmptyString Then Throw New ArgumentNullException("format", nullMsg) j = JsonDocument.Parse(AuthFile.GetText)
pattern = pattern.Replace("{}", "{0}").Replace("{:x}", "{1:x}") Catch jex As Exception
If Round = 0 Then
AuthFile.Delete()
UpdateAuthFile(True)
Return UpdateSignature(Path, ForceUpdateAuth, Round + 1)
Else
MySettings.SessionAborted = True
Return False
End If
End Try
If Not j Is Nothing Then
Dim pattern$ = j.Value("format")
If pattern.IsEmptyString Then Throw New ArgumentNullException("format", nullMsg)
pattern = pattern.Replace("{}", "{0}").Replace("{:x}", "{1:x}")
Dim li%() = j("checksum_indexes").Select(Function(e) CInt(e(0).Value)).ToArray Dim li%() = j("checksum_indexes").Select(Function(e) CInt(e(0).Value)).ToArray
If Not li.ListExists Then Throw New ArgumentNullException("checksum_indexes", nullMsg) If Not li.ListExists Then Throw New ArgumentNullException("checksum_indexes", nullMsg)
If j.Value("static_param").IsEmptyString Then Throw New ArgumentNullException("static_param", nullMsg) If j.Value("static_param").IsEmptyString Then Throw New ArgumentNullException("static_param", nullMsg)
If j.Value("checksum_constant").IsEmptyString Then Throw New ArgumentNullException("checksum_constant", nullMsg) If j.Value("checksum_constant").IsEmptyString Then Throw New ArgumentNullException("checksum_constant", nullMsg)
Dim t$ = ADateTime.ConvertToUnix64(Now.ToUniversalTime).ToString Dim t$ = ADateTime.ConvertToUnix64(Now.ToUniversalTime).ToString
Dim h$ = String.Join(vbLf, j.Value("static_param"), t, Path, MySettings.HH_USER_ID.Value.ToString) Dim h$ = String.Join(vbLf, j.Value("static_param"), t, Path, MySettings.HH_USER_ID.Value.ToString)
Dim hash$ = GetHashSha1(h) Dim hash$ = GetHashSha1(h)
Dim hashBytes() As Byte = System.Text.Encoding.ASCII.GetBytes(hash) Dim hashBytes() As Byte = System.Text.Encoding.ASCII.GetBytes(hash)
Dim hashSum% = li.Sum(Function(i) hashBytes(i)) + CInt(j.Value("checksum_constant")) Dim hashSum% = li.Sum(Function(i) hashBytes(i)) + CInt(j.Value("checksum_constant"))
Dim sign$ = String.Format(pattern, hash, Math.Abs(hashSum)) Dim sign$ = String.Format(pattern, hash, Math.Abs(hashSum))
'#If DEBUG Then '#If DEBUG Then
'Debug.WriteLine(sign) 'Debug.WriteLine(sign)
'Debug.WriteLine(t) 'Debug.WriteLine(t)
'#End If '#End If
Responser.Headers.Add(HeaderSign, sign) Responser.Headers.Add(HeaderSign, sign)
Responser.Headers.Add(HeaderTime, t) Responser.Headers.Add(HeaderTime, t)
j.Dispose() j.Dispose()
Return True Return True
Else End If
Return False
End If End If
Return False
Catch ex As Exception Catch ex As Exception
Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, $"{ToStringForLog()}: UpdateSignature", False) Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, $"{ToStringForLog()}: UpdateSignature", False)
End Try End Try
@@ -526,7 +540,7 @@ Namespace API.OnlyFans
Private _DownloadingException_AuthFileUpdate As Boolean = False Private _DownloadingException_AuthFileUpdate As Boolean = False
Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False, Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False,
Optional ByVal EObj As Object = Nothing) As Integer Optional ByVal EObj As Object = Nothing) As Integer
If Responser.StatusCode = Net.HttpStatusCode.BadRequest Then If Responser.StatusCode = Net.HttpStatusCode.BadRequest Then '400
If Not _DownloadingException_AuthFileUpdate AndAlso UpdateAuthFile(True) Then If Not _DownloadingException_AuthFileUpdate AndAlso UpdateAuthFile(True) Then
_DownloadingException_AuthFileUpdate = True _DownloadingException_AuthFileUpdate = True
Return 2 Return 2
@@ -535,13 +549,17 @@ Namespace API.OnlyFans
MyMainLOG = $"{ToStringForLog()}: OnlyFans credentials expired" MyMainLOG = $"{ToStringForLog()}: OnlyFans credentials expired"
Return 1 Return 1
End If End If
ElseIf Responser.StatusCode = Net.HttpStatusCode.NotFound Then ElseIf Responser.StatusCode = Net.HttpStatusCode.NotFound Then '404
UserExists = False UserExists = False
Return 1 Return 1
ElseIf Responser.StatusCode = Net.HttpStatusCode.GatewayTimeout Or Responser.StatusCode = 429 Then ElseIf Responser.StatusCode = Net.HttpStatusCode.GatewayTimeout Or Responser.StatusCode = 429 Then '504, 429
If Responser.StatusCode = 429 Then MyMainLOG = $"[429] OnlyFans too many requests ({ToStringForLog()})" If Responser.StatusCode = 429 Then MyMainLOG = $"[429] OnlyFans too many requests ({ToStringForLog()})"
MySettings.SessionAborted = True MySettings.SessionAborted = True
Return 1 Return 1
ElseIf Responser.StatusCode = Net.HttpStatusCode.Unauthorized Then '401
MySettings.SessionAborted = True
MyMainLOG = $"{ToStringForLog()}: OnlyFans credentials expired"
Return 1
Else Else
Return 0 Return 0
End If End If

3
SCrawler/API/Xhamster/UserData.vb
View File

@@ -582,7 +582,8 @@ Namespace API.Xhamster
#Region "Exception" #Region "Exception"
Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False, Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False,
Optional ByVal EObj As Object = Nothing) As Integer Optional ByVal EObj As Object = Nothing) As Integer
Return If(Responser.Status = Net.WebExceptionStatus.ConnectionClosed, 1, 0) '8, 503
Return If(Responser.Status = Net.WebExceptionStatus.ConnectionClosed Or Responser.StatusCode = Net.HttpStatusCode.ServiceUnavailable, 1, 0)
End Function End Function
#End Region #End Region
#Region "IDisposable support" #Region "IDisposable support"

29
SCrawler/Download/Automation/Scheduler.vb
View File

@@ -130,16 +130,25 @@ Namespace DownloadObjects
#End Region #End Region
#Region "Execution" #Region "Execution"
Friend Async Function Start(ByVal Init As Boolean) As Task Friend Async Function Start(ByVal Init As Boolean) As Task
Await Task.Run(Sub() Try
If Count > 0 Then Await Task.Run(Sub()
If Plans.Exists(PlanDownloading) Then PlansWaiter(PlanDownloading) If Count > 0 Then
For Each Plan In Plans If Plans.Exists(PlanDownloading) Then PlansWaiter(PlanDownloading)
Plan.Start(Init) For Each Plan In Plans
PlansWaiter(PlanDownloading) Plan.Start(Init)
Thread.Sleep(1000) PlansWaiter(PlanDownloading)
Next Thread.Sleep(1000)
End If Next
End Sub) End If
End Sub)
Catch ex As Exception
If Init Then
ErrorsDescriber.Execute(EDP.SendToLog, ex, "Start automation")
MainFrameObj.UpdateLogButton()
Else
Throw ex
End If
End Try
End Function End Function
Friend Sub [Stop]() Friend Sub [Stop]()
If Count > 0 Then Plans.ForEach(Sub(p) p.Stop()) If Count > 0 Then Plans.ForEach(Sub(p) p.Stop())

5
SCrawler/Download/TDownloader.vb
View File

@@ -156,9 +156,10 @@ Namespace DownloadObjects
Private ReadOnly NProv As IFormatProvider Private ReadOnly NProv As IFormatProvider
#End Region #End Region
#Region "Working, Count" #Region "Working, Count"
Friend ReadOnly Property Working As Boolean Friend ReadOnly Property Working(Optional ByVal CheckThread As Boolean = True) As Boolean
Get Get
Return _PoolReconfiguration Or (Pool.Count > 0 AndAlso Pool.Exists(Function(j) j.Working)) Or If(CheckerThread?.IsAlive, False) Return _PoolReconfiguration Or (Pool.Count > 0 AndAlso Pool.Exists(Function(j) j.Working)) Or
(CheckThread AndAlso If(CheckerThread?.IsAlive, False))
End Get End Get
End Property End Property
Friend ReadOnly Property Count As Integer Friend ReadOnly Property Count As Integer

15
SCrawler/MainFrame.vb
View File

@@ -578,10 +578,15 @@ CloseResume:
ControlInvokeFast(Me, Sub() BTT_TRAY_PAUSE_AUTOMATION.Visible = b) ControlInvokeFast(Me, Sub() BTT_TRAY_PAUSE_AUTOMATION.Visible = b)
End Sub End Sub
Private Async Sub BTT_DOWN_AUTOMATION_Click(sender As Object, e As EventArgs) Handles BTT_DOWN_AUTOMATION.Click Private Async Sub BTT_DOWN_AUTOMATION_Click(sender As Object, e As EventArgs) Handles BTT_DOWN_AUTOMATION.Click
Using f As New SchedulerEditorForm : f.ShowDialog() : End Using Try
Await Settings.Automation.Start(False) Using f As New SchedulerEditorForm : f.ShowDialog() : End Using
UpdatePauseButtonsVisibility() Await Settings.Automation.Start(False)
MainFrameObj.PauseButtons.UpdatePauseButtons() UpdatePauseButtonsVisibility()
MainFrameObj.PauseButtons.UpdatePauseButtons()
Catch ex As Exception
ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Start automation")
MainFrameObj.UpdateLogButton()
End Try
End Sub End Sub
Private Sub BTT_DOWN_AUTOMATION_PAUSE_Click(sender As Object, e As EventArgs) Handles BTT_DOWN_AUTOMATION_PAUSE.Click, BTT_TRAY_PAUSE_AUTOMATION.Click Private Sub BTT_DOWN_AUTOMATION_PAUSE_Click(sender As Object, e As EventArgs) Handles BTT_DOWN_AUTOMATION_PAUSE.Click, BTT_TRAY_PAUSE_AUTOMATION.Click
Dim p As PauseModes = Settings.Automation.Pause Dim p As PauseModes = Settings.Automation.Pause
@@ -1981,7 +1986,7 @@ ResumeDownloadingOperation:
MainFrameObj.UpdateLogButton() MainFrameObj.UpdateLogButton()
End Sub End Sub
Private Sub Downloader_Downloading(ByVal Value As Boolean) Private Sub Downloader_Downloading(ByVal Value As Boolean)
Dim __isDownloading As Boolean = Value Or Downloader.Working Dim __isDownloading As Boolean = Value Or Downloader.Working(False)
ControlInvokeFast(Toolbar_TOP, BTT_DOWN_STOP, Sub() BTT_DOWN_STOP.Enabled = __isDownloading) ControlInvokeFast(Toolbar_TOP, BTT_DOWN_STOP, Sub() BTT_DOWN_STOP.Enabled = __isDownloading)
TrayIcon.Icon = If(__isDownloading, My.Resources.ArrowDownIcon_Blue_24, My.Resources.RainbowIcon_48) TrayIcon.Icon = If(__isDownloading, My.Resources.ArrowDownIcon_Blue_24, My.Resources.RainbowIcon_48)
End Sub End Sub