geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-15 16:22:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

2022.8.22.0

Browse Source 
Cleaned up the code
Replace some old functions with new ones
Adapted to the new library environment
Enable/Disable display user/downloaded image
Autodownloader option 'Show notification' not saved
Separate thread for standalone video downloader
Expanded the description of some errors with additional information
Fixed date/time renaming issue
Fixed internal library bugs
Fixed minor bugs
This commit is contained in:
Andy
2022-08-22 02:42:36 +03:00
parent e0dc66e0da
commit 54ffe10f71
88 changed files with 1155 additions and 1066 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
SCrawler/API/Twitter/SiteSettings.vb
View File

@@ -9,7 +9,7 @@
Imports SCrawler.API.Base
Imports SCrawler.Plugin
Imports SCrawler.Plugin.Attributes
Imports PersonalUtilities.Tools
Imports PersonalUtilities.Tools.WEB
Imports PersonalUtilities.Functions.RegularExpressions
Namespace API.Twitter
<Manifest("AndyProgram_Twitter"), SavedPosts, UseClassAsIs>
@@ -33,10 +33,10 @@ Namespace API.Twitter
Private ReadOnly Property Token As PropertyValue
<PropertyOption(ControlText:="Saved posts user name", ControlToolTip:="Personal profile username", LeftOffset:=120), PXML>
Friend ReadOnly Property SavedPostsUserName As PropertyValue
Friend Overrides ReadOnly Property Responser As WEB.Response
Friend Overrides ReadOnly Property Responser As Response
Friend Sub New()
MyBase.New(TwitterSite)
Responser = New WEB.Response($"{SettingsFolderName}\Responser_{Site}.xml")
Responser = New Response($"{SettingsFolderName}\Responser_{Site}.xml")
Dim a$ = String.Empty
Dim t$ = String.Empty
@@ -52,11 +52,10 @@ Namespace API.Twitter
.ContentType = "application/json"
.Accept = "*/*"
.CookiesDomain = "twitter.com"
.Cookies = New CookieKeeper(.CookiesDomain)
.Decoders.Add(SymbolsConverter.Converters.Unicode)
With .Headers
.Add("sec-ch-ua", " Not;A Brand" & Chr(34) & ";v=" & Chr(34) & "99" & Chr(34) & ", " & Chr(34) &
"Google Chrome" & Chr(34) & ";v=" & Chr(34) & "91" & Chr(34) & ", " & Chr(34) & "Chromium" &
Chr(34) & ";v=" & Chr(34) & "91" & Chr(34))
.Add("sec-ch-ua", " Not;A Brand"";v=""99"", ""Google Chrome"";v=""91"", ""Chromium"";v=""91""")
.Add("sec-ch-ua-mobile", "?0")
.Add("sec-fetch-dest", "empty")
.Add("sec-fetch-mode", "cors")

14
SCrawler/API/Twitter/UserData.vb
View File

@@ -63,7 +63,7 @@ Namespace API.Twitter
Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException)
If Not r.IsEmptyString Then
Using w As EContainer = JsonDocument.Parse(r)
If Not w Is Nothing AndAlso w.Count > 0 Then
If w.ListExists Then
For Each nn In If(IsSavedPosts, w({"globalObjects", "tweets"}).XmlIfNothing, w)
ThrowAny(Token)
If nn.Count > 0 Then
@@ -141,8 +141,7 @@ Namespace API.Twitter
If URL.Contains("twitter") Then
Dim PostID$ = RegexReplace(URL, RParams.DM("(?<=/)\d+", 0))
If Not PostID.IsEmptyString Then
Dim r$ = DirectCast(resp.Copy(), Response).
GetResponse($"https://api.twitter.com/1.1/statuses/show.json?id={PostID}",, EDP.ReturnValue)
Dim r$ = DirectCast(resp.Copy(), Response).GetResponse($"https://api.twitter.com/1.1/statuses/show.json?id={PostID}",, EDP.ReturnValue)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r)
If j.ListExists Then
@@ -163,7 +162,7 @@ Namespace API.Twitter
Const P4K As String = "4096x4096"
Try
Dim ww As EContainer = w("sizes")
If Not ww Is Nothing AndAlso ww.Count > 0 Then
If ww.ListExists Then
Dim l As New List(Of Sizes)
Dim Orig As Sizes? = New Sizes(w.Value({"original_info"}, "height").FromXML(Of Integer)(-1), P4K)
If Orig.Value.Value = -1 Then Orig = Nothing
@@ -177,7 +176,6 @@ Namespace API.Twitter
Return P4K
ElseIf l(0).Data.IsEmptyString Then
Return P4K
'If LargeContained Then Return "large" Else Return P4K
Else
Return l(0).Data
End If
@@ -222,8 +220,8 @@ Namespace API.Twitter
Dim url$, ff$
Dim f As SFile
Dim m As UserMedia
With w({"extended_entities", "media"}).XmlIfNothing
If .Count > 0 Then
With w({"extended_entities", "media"})
If .ListExists Then
For Each n As EContainer In .Self
If n.Value("type") = "animated_gif" Then
With n({"video_info", "variants"}).XmlIfNothing.ItemF({gifUrl}).XmlIfNothing
@@ -251,7 +249,7 @@ Namespace API.Twitter
End Function
Private Shared Function GetVideoNodeURL(ByVal w As EContainer) As String
Dim v As EContainer = w.GetNode(VideoNode)
If Not v Is Nothing AndAlso v.Count > 0 Then
If v.ListExists Then
Dim l As New List(Of Sizes)
Dim u$
Dim nn As EContainer