geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-15 08:12:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

2023.11.24.0

Browse Source 
Scheduler: handle scheduler change error (collection was modified)
SchedulerEditorForm: add scheduler name to form title
DownloadProgress: fix disposing error when some objects are already null
API.Reddit: add special notification for error 429
API.Twitter: handle JSON deserialization error
Porn sites: fix incorrect parsing of search queries
YouTube: path not set when adding array to download
This commit is contained in:
Andy
2023-11-24 04:54:04 +03:00
parent 0ec617c1dc
commit a540aded68
20 changed files with 157 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
SCrawler/API/Twitter/UserData.vb
View File

@@ -12,6 +12,7 @@ Imports SCrawler.API.YouTube.Objects
Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.RegularExpressions
Imports PersonalUtilities.Tools
Imports PersonalUtilities.Tools.Web.Documents
Imports PersonalUtilities.Tools.Web.Documents.JSON
Imports UStates = SCrawler.API.Base.UserMedia.States
Imports UTypes = SCrawler.API.Base.UserMedia.Types
@@ -154,6 +155,7 @@ Namespace API.Twitter
Private Sub DownloadData_Timeline(ByVal Token As CancellationToken)
Dim URL$ = String.Empty
Dim tCache As CacheKeeper = Nothing
Dim jsonArgs As New WebDocumentEventArgs With {.DeclaredError = EDP.ThrowException}
Try
Const entry$ = "entry"
Dim PostID$ = String.Empty
@@ -231,7 +233,8 @@ Namespace API.Twitter
For i = 0 To timelineFiles.Count - 1 : timelineFiles(i) = RenameGdlFile(timelineFiles(i), i) : Next
'parse files
For i = 0 To timelineFiles.Count - 1
j = JsonDocument.Parse(timelineFiles(i).GetText)
j = JsonDocument.Parse(timelineFiles(i).GetText, jsonArgs)
jsonArgs.Reset()
If Not j Is Nothing Then
If i = 0 Then
If Not userInfoParsed Then
@@ -339,12 +342,15 @@ Namespace API.Twitter
End If
DownloadModelForceApply = False
FirstDownloadComplete = True
Catch jsonNull_ex As ArgumentNullException When jsonArgs.State = WebDocumentEventArgs.States.Error
Throw New Plugin.ExitException($"{ToStringForLog()}: No deserialized data found")
Catch limit_ex As TwitterLimitException
Throw limit_ex
Catch ex As Exception
ProcessException(ex, Token, $"data downloading error [{URL}]")
Finally
If Not tCache Is Nothing Then tCache.Dispose()
jsonArgs.DisposeIfReady
If _TempPostsList.Count > 0 Then _TempPostsList.Sort()
End Try
End Sub