geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-15 00:02:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

1.0.0.1

Browse Source 
Added: limited downloads, remove active downloads, simulation tasks management, copy channel image, x86
Removed reparse function
Fixed some errors, typo and Reddit parsing algo
This commit is contained in:
Andy
2021-12-09 18:36:22 +03:00
parent 6979ca018e
commit c7dc0eca7d
23 changed files with 642 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
SCrawler/API/Twitter/UserData.vb
View File

@@ -58,13 +58,13 @@ Namespace API.Twitter
URL = $"https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name={Name}&count=200&exclude_replies=false&include_rts=1&tweet_mode=extended"
If Not POST.IsEmptyString Then URL &= $"&max_id={POST}"
Token.ThrowIfCancellationRequested()
ThrowAny(Token)
Dim r$ = Settings.Site(Sites.Twitter).Responser.GetResponse(URL,, EDP.ThrowException)
If Not r.IsEmptyString Then
Using w As EContainer = JsonDocument.Parse(r)
If Not w Is Nothing AndAlso w.Count > 0 Then
For Each nn In w
Token.ThrowIfCancellationRequested()
ThrowAny(Token)
If nn.Count > 0 Then
PostID = nn.Value("id")
If ID.IsEmptyString Then
@@ -107,6 +107,7 @@ Namespace API.Twitter
If Not PostID.IsEmptyString And NewPostDetected Then DownloadData(PostID, Token)
End If
Catch oex As OperationCanceledException When Token.IsCancellationRequested
Catch dex As ObjectDisposedException When Disposed
Catch ex As Exception
LogError(ex, $"data downloading error [{URL}]")
HasError = True
@@ -206,7 +207,7 @@ Namespace API.Twitter
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
Try
Dim i%
Token.ThrowIfCancellationRequested()
ThrowAny(Token)
If _ContentNew.Count > 0 Then
_ContentNew.RemoveAll(Function(c) c.URL.IsEmptyString)
If _ContentNew.Count > 0 Then
@@ -219,7 +220,7 @@ Namespace API.Twitter
If vsf Then SFileShares.SFileExists($"{MyDir}\Video\", SFO.Path)
MainProgress.TotalCount += _ContentNew.Count
For i = 0 To _ContentNew.Count - 1
Token.ThrowIfCancellationRequested()
ThrowAny(Token)
v = _ContentNew(i)
v.State = UStates.Tried
If v.File.IsEmptyString Then
@@ -255,6 +256,7 @@ Namespace API.Twitter
End If
End If
Catch oex As OperationCanceledException When Token.IsCancellationRequested
Catch dex As ObjectDisposedException When Disposed
Catch ex As Exception
LogError(ex, "content downloading error")
HasError = True