mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-18 09:42:18 +00:00
2025.2.25.0
PluginProvider IPluginContentProvider: add 'NameTrue' property YT YouTubeSettings: remove the 'CreateDescriptionFiles_AddUploadDate' property PlayListParserForm: add 'RDAMP' as default value when initializing form YouTubeMediaContainerBase: fix line breaks SCrawler API.Base: add 'EditorExchangeOptionsBase' class API.Base.GDL: move functions to Pinterest.UserData API.Base.IUserData: add 'NameTrue' property API.Base.SiteSettingsBase: update the 'GetUserUrl' function to use the 'NameTrue' property; update 'UserOptions' function API.Base.UserDataBase: add 'NameTrue' property; add 'SimpleDownloadAvatar' function to get rid of the same functions of other classes; Update 'UserDescriptionUpdate' function API.Base.InternalSettingsForm: calculate max offset ADD API.Bluesky API.Facebook: add 'Reels' downloads; fix video downloading API.Instagram: add inheritance 'EditorExchangeOptionsBase'; remove 'GetUserUrl' function from 'SiteSettings'; update 'UserData' class to new environment; add saving 'heic' file along with 'jpg' API.LPSG.UserData: simplify 403 error API.Mastodon: update classes to new environment API.OnlyFans: add 'AppTokenDefault'; disable cookies update; update 'UserData' class to new environment API.Pinterest: add sub-boards downloading; update download functions API.PornHub: fix photo & video downloading; remove 'ModelHub' support API.Reddit: fix token update; update 'UserData' class to new environment API.ThisVid: update 'UserData' class to new environment API.ThreadsNet: fix data download; update classes to new environment; fix 'UserID' extraction; add the ability to manually change the UserName API.TikTok: update classes to new environment API.Twitter: update classes to new environment; add sleep timers to fully download large profiles; add 'CookiesUpdate' hidden property API.Xhamster: update classes to new environment API.XVIDEOS: update classes to new environment Feed: add the ability to invert selection; open post URL when double-clicking on subscription image FeedSpecialCollection: update 'FeedsComparer' GlobalSettingsForm: remove 'UserAgent' from the 'Basis' tab UserDataHost: update class to new environment SettingsCLS: set the 'UserSiteNameAsFriendly' property to 'True' by default; disable 'DownDetector'; add 'UsersListProtected' property
This commit is contained in:
Andy
@@ -7,6 +7,7 @@
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports System.Threading
|
||||
Imports SCrawler.Plugin
|
||||
Imports SCrawler.API.Base
|
||||
Imports SCrawler.API.YouTube.Objects
|
||||
Imports PersonalUtilities.Functions.XML
|
||||
@@ -51,9 +52,10 @@ Namespace API.ThreadsNet
|
||||
#End Region
|
||||
#Region "Exchange"
|
||||
Friend Overrides Function ExchangeOptionsGet() As Object
|
||||
Return Nothing
|
||||
Return New EditorExchangeOptionsBase(Me)
|
||||
End Function
|
||||
Friend Overrides Sub ExchangeOptionsSet(ByVal Obj As Object)
|
||||
If Not Obj Is Nothing AndAlso TypeOf Obj Is EditorExchangeOptionsBase Then NameTrue = DirectCast(Obj, EditorExchangeOptionsBase).UserName
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Initializer"
|
||||
@@ -73,12 +75,13 @@ Namespace API.ThreadsNet
|
||||
End Sub
|
||||
Private Sub DisableDownload()
|
||||
MySettings.DownloadData_Impl.Value = False
|
||||
MyMainLOG = $"{Site} downloading is disabled until you update your credentials"
|
||||
LogError(Nothing, $"{Site} downloading is disabled until you update your credentials")
|
||||
End Sub
|
||||
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
|
||||
If CBool(MySettings.DownloadData_Impl.Value) Then
|
||||
Dim errorFound As Boolean = False
|
||||
Try
|
||||
_IdChanged = False
|
||||
Responser.Method = "POST"
|
||||
LoadSavePostsKV(True)
|
||||
ResetBaseTokens()
|
||||
@@ -103,9 +106,9 @@ Namespace API.ThreadsNet
|
||||
End If
|
||||
If IsSavedPosts Then
|
||||
DefaultParser_ElemNode = {"node", "thread_items", 0, "post"}
|
||||
DownloadSavedPosts(String.Empty, Token)
|
||||
DownloadSavedPosts(String.Empty, 0, Token)
|
||||
Else
|
||||
DownloadData(String.Empty, Token)
|
||||
DownloadData(String.Empty, 0, Token)
|
||||
End If
|
||||
If _TempMediaList.Count > 0 Then FirstLoadingDone = True : setMaxPostDate.Invoke(_TempMediaList)
|
||||
Catch ex As Exception
|
||||
@@ -151,12 +154,14 @@ Namespace API.ThreadsNet
|
||||
Responser.Headers.Add(IGS.Header_CSRF_TOKEN, csrf)
|
||||
End If
|
||||
End Sub
|
||||
Private Const GQL_Q As String = "https://www.threads.net/api/graphql?lsd={0}&fb_dtsg={1}&doc_id={2}&fb_api_req_friendly_name={3}&server_timestamps=true&variables={4}"
|
||||
Private Const GQL_P_DOC_ID As String = "6371597506283707"
|
||||
Private Const GQL_P_NAME As String = "BarcelonaProfileThreadsTabRefetchableQuery"
|
||||
Private Const GQL_S_DOC_ID_1 As String = "7758166704280174"
|
||||
Private Const GQL_S_NAME_1 As String = "BarcelonaSavedPageViewerQuery"
|
||||
Private Const GQL_S_DOC_ID_2 As String = "8617275414954442"
|
||||
'Private Const GQL_Q As String = "https://www.threads.net/api/graphql?lsd={0}&fb_dtsg={1}&doc_id={2}&fb_api_req_friendly_name={3}&server_timestamps=true&variables={4}"
|
||||
Private Const GQL_Q2 As String = "https://www.threads.net/graphql/query"
|
||||
Private Const PayloadData As String = "lsd={0}&fb_dtsg={1}&doc_id={2}&fb_api_req_friendly_name={3}&server_timestamps=true&variables={4}"
|
||||
Private Const GQL_P_DOC_ID As String = "9039187972876777" '"8779269398849532" '"6371597506283707"
|
||||
Private Const GQL_P_NAME As String = "BarcelonaProfileThreadsTabRefetchableDirectQuery" '"BarcelonaProfileThreadsTabRefetchableQuery"
|
||||
'Private Const GQL_S_DOC_ID_1 As String = "9227844190587889" '"7758166704280174"
|
||||
'Private Const GQL_S_NAME_1 As String = "BarcelonaSavedPageViewerQuery"
|
||||
Private Const GQL_S_DOC_ID_2 As String = "9116629201788321" '"8617275414954442"
|
||||
Private Const GQL_S_NAME_2 As String = "BarcelonaSavedPageRefetchableQuery"
|
||||
Private Sub DownloadCheckCredentials()
|
||||
If Not Valid Then
|
||||
@@ -166,106 +171,189 @@ Namespace API.ThreadsNet
|
||||
End If
|
||||
If Not Valid Then DisableDownload() : Throw New Plugin.ExitException("Some credentials are missing")
|
||||
End Sub
|
||||
Private Overloads Sub DownloadData(ByVal Cursor As String, ByVal Token As CancellationToken)
|
||||
Const var_init$ = """userID"":""{0}"""
|
||||
Const var_cursor$ = """after"":""{1}"",""before"":null,""first"":25,""last"":null,""userID"":""{0}"",""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsFeedbackHubEnabledrelayprovider"":false"
|
||||
Dim URL$ = String.Empty
|
||||
Private Function CheckErrors(ByVal e As EContainer) As Boolean
|
||||
Return e.ListExists AndAlso Not JsonErrorMessage(e).IsEmptyString
|
||||
End Function
|
||||
Private Function JsonErrorMessage(ByVal e As EContainer) As String
|
||||
Return e.ItemF({"errors", 0, "summary"})?.Value
|
||||
End Function
|
||||
Private Sub ProcessJsonErrorException(ByVal uex As JsonErrorException, Optional ByVal ThrowEx As Boolean = True)
|
||||
If uex.UserNotFound Then
|
||||
UserExists = False
|
||||
_ForceSaveUserInfo = True
|
||||
_ForceSaveUserInfoOnException = True
|
||||
ElseIf ThrowEx Then
|
||||
Throw New ExitException(uex.ErrMessage) With {.SimpleLogLine = True}
|
||||
Else
|
||||
LogError(Nothing, uex.ErrMessage)
|
||||
End If
|
||||
End Sub
|
||||
Private Class JsonErrorException : Inherits Exception
|
||||
Friend Property UserNotFound As Boolean = False
|
||||
Private _ErrMessage As String = String.Empty
|
||||
Public Property ErrMessage As String
|
||||
Get
|
||||
Return _ErrMessage
|
||||
End Get
|
||||
Set(ByVal m As String)
|
||||
_ErrMessage = m
|
||||
UserNotFound = _ErrMessage.StringToLower = "not found"
|
||||
End Set
|
||||
End Property
|
||||
Public Overrides ReadOnly Property Message As String
|
||||
Get
|
||||
Return _ErrMessage
|
||||
End Get
|
||||
End Property
|
||||
Friend Sub New()
|
||||
End Sub
|
||||
Friend Sub New(ByVal Message As String)
|
||||
ErrMessage = Message
|
||||
End Sub
|
||||
End Class
|
||||
Private Overloads Sub DownloadData(ByVal Cursor As String, ByVal Round As Integer, ByVal Token As CancellationToken)
|
||||
'Const var_init$ = """userID"":""{0}"""
|
||||
'Const var_cursor$ = """after"":""{1}"",""before"":null,""first"":25,""last"":null,""userID"":""{0}"",""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsFeedbackHubEnabledrelayprovider"":false"
|
||||
Const var_cursor2$ = """after"":{1},""before"":null,""first"":10,""last"":null,""userID"":""{0}"",""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaHasSelfReplyContextrelayprovider"":false,""__relay_internal__pv__BarcelonaShareableListsrelayprovider"":true,""__relay_internal__pv__BarcelonaIsSearchDiscoveryEnabledrelayprovider"":false,""__relay_internal__pv__BarcelonaOptionalCookiesEnabledrelayprovider"":true,""__relay_internal__pv__BarcelonaQuotedPostUFIEnabledrelayprovider"":false,""__relay_internal__pv__BarcelonaIsCrawlerrelayprovider"":false,""__relay_internal__pv__BarcelonaHasDisplayNamesrelayprovider"":false,""__relay_internal__pv__BarcelonaCanSeeSponsoredContentrelayprovider"":false,""__relay_internal__pv__BarcelonaShouldShowFediverseM075Featuresrelayprovider"":true,""__relay_internal__pv__BarcelonaShouldShowTagRedesignrelayprovider"":false,""__relay_internal__pv__BarcelonaIsInternalUserrelayprovider"":false"
|
||||
Try
|
||||
DownloadCheckCredentials()
|
||||
|
||||
Responser.Method = "POST"
|
||||
Responser.Referer = $"https://www.threads.net/@{NameTrue}"
|
||||
Responser.Headers.Add(GQL_HEADER_FB_LSD, Token_lsd)
|
||||
Responser.Headers.Add(GQL_HEADER_FB_FRINDLY_NAME, GQL_P_NAME)
|
||||
With Responser
|
||||
.Method = "POST"
|
||||
.Referer = $"https://www.threads.net/@{NameTrue}"
|
||||
.ContentType = "application/x-www-form-urlencoded"
|
||||
With .Headers
|
||||
.Add(GQL_HEADER_FB_LSD, Token_lsd)
|
||||
.Add(GQL_HEADER_FB_FRINDLY_NAME, GQL_P_NAME)
|
||||
End With
|
||||
End With
|
||||
|
||||
Dim nextCursor$ = String.Empty
|
||||
Dim dataFound As Boolean = False
|
||||
|
||||
Dim vars$
|
||||
If Cursor.IsEmptyString Then
|
||||
vars = String.Format(var_init, ID)
|
||||
Else
|
||||
vars = String.Format(var_cursor, ID, Cursor)
|
||||
End If
|
||||
vars = SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & vars & "}")
|
||||
Dim vars$ = String.Format(var_cursor2, ID, IIf(Cursor.IsEmptyString, "null", $"""{Cursor}"""))
|
||||
'If Cursor.IsEmptyString Then
|
||||
' vars = String.Format(var_init, ID)
|
||||
'Else
|
||||
' vars = String.Format(var_cursor, ID, Cursor)
|
||||
'End If
|
||||
vars = String.Format(PayloadData, Token_lsd, Token_dtsg_Var, GQL_P_DOC_ID, GQL_P_NAME,
|
||||
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & vars & "}"))
|
||||
|
||||
URL = String.Format(GQL_Q, Token_lsd, Token_dtsg_Var, GQL_P_DOC_ID, GQL_P_NAME, vars)
|
||||
'URL = String.Format(GQL_Q, Token_lsd, Token_dtsg_Var, GQL_P_DOC_ID, GQL_P_NAME, vars)
|
||||
|
||||
Using j As EContainer = GetDocument(URL, Token)
|
||||
If j.ListExists Then
|
||||
With j({"data", "mediaData"})
|
||||
If .ListExists Then
|
||||
nextCursor = .Value({"page_info"}, "end_cursor")
|
||||
With .Item({"edges"})
|
||||
If .ListExists Then dataFound = DefaultParser(.Self, Sections.Timeline, Token)
|
||||
End With
|
||||
End If
|
||||
End With
|
||||
Using j As EContainer = GetDocument(GQL_Q2, vars, Token)
|
||||
If Not CheckErrors(j) Then
|
||||
If j.ListExists Then
|
||||
With j({"data", "mediaData"})
|
||||
If .ListExists Then
|
||||
nextCursor = .Value({"page_info"}, "end_cursor")
|
||||
With .Item({"edges"})
|
||||
If .ListExists Then dataFound = DefaultParser(.Self, Sections.Timeline, Token)
|
||||
End With
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
Else
|
||||
Throw New JsonErrorException(JsonErrorMessage(j))
|
||||
End If
|
||||
End Using
|
||||
|
||||
If dataFound And Not nextCursor.IsEmptyString Then DownloadData(nextCursor, Token)
|
||||
If dataFound And Not nextCursor.IsEmptyString Then DownloadData(nextCursor, 0, Token)
|
||||
Catch uex As JsonErrorException
|
||||
If Round > 0 Then
|
||||
ProcessJsonErrorException(uex)
|
||||
ElseIf Not _IdChanged AndAlso UpdateCredentials() Then
|
||||
DownloadData(Cursor, Round + 1, Token)
|
||||
Else
|
||||
ProcessJsonErrorException(uex)
|
||||
End If
|
||||
Catch eex As ExitException
|
||||
Throw eex
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, $"data downloading error [{URL}]")
|
||||
ProcessException(ex, Token, "data downloading error")
|
||||
End Try
|
||||
End Sub
|
||||
Private Sub DownloadSavedPosts(ByVal Cursor As String, ByVal Token As CancellationToken)
|
||||
Const var_init$ = """__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsInlineReelsEnabledrelayprovider"":false,""__relay_internal__pv__BarcelonaUseCometVideoPlaybackEnginerelayprovider"":false,""__relay_internal__pv__BarcelonaOptionalCookiesEnabledrelayprovider"":true,""__relay_internal__pv__BarcelonaIsTextFragmentsEnabledForPostCaptionsrelayprovider"":true,""__relay_internal__pv__BarcelonaShouldShowFediverseM075Featuresrelayprovider"":true"
|
||||
Const var_cursor$ = """after"":""{0}"",""first"":25,""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsInlineReelsEnabledrelayprovider"":false,""__relay_internal__pv__BarcelonaUseCometVideoPlaybackEnginerelayprovider"":false,""__relay_internal__pv__BarcelonaOptionalCookiesEnabledrelayprovider"":true,""__relay_internal__pv__BarcelonaIsTextFragmentsEnabledForPostCaptionsrelayprovider"":true,""__relay_internal__pv__BarcelonaShouldShowFediverseM075Featuresrelayprovider"":true"
|
||||
Dim URL$ = String.Empty
|
||||
Private Sub DownloadSavedPosts(ByVal Cursor As String, ByVal Round As Integer, ByVal Token As CancellationToken)
|
||||
'Const var_init$ = """__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsInlineReelsEnabledrelayprovider"":false,""__relay_internal__pv__BarcelonaUseCometVideoPlaybackEnginerelayprovider"":false,""__relay_internal__pv__BarcelonaOptionalCookiesEnabledrelayprovider"":true,""__relay_internal__pv__BarcelonaIsTextFragmentsEnabledForPostCaptionsrelayprovider"":true,""__relay_internal__pv__BarcelonaShouldShowFediverseM075Featuresrelayprovider"":true"
|
||||
'Const var_cursor$ = """after"":""{0}"",""first"":25,""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsInlineReelsEnabledrelayprovider"":false,""__relay_internal__pv__BarcelonaUseCometVideoPlaybackEnginerelayprovider"":false,""__relay_internal__pv__BarcelonaOptionalCookiesEnabledrelayprovider"":true,""__relay_internal__pv__BarcelonaIsTextFragmentsEnabledForPostCaptionsrelayprovider"":true,""__relay_internal__pv__BarcelonaShouldShowFediverseM075Featuresrelayprovider"":true"
|
||||
Const var_cursor2$ = """after"":{0},""first"":25,""__relay_internal__pv__BarcelonaQuotedPostUFIEnabledrelayprovider"":false,""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaHasSelfReplyContextrelayprovider"":false,""__relay_internal__pv__BarcelonaShareableListsrelayprovider"":true,""__relay_internal__pv__BarcelonaIsSearchDiscoveryEnabledrelayprovider"":false,""__relay_internal__pv__BarcelonaOptionalCookiesEnabledrelayprovider"":true,""__relay_internal__pv__BarcelonaIsCrawlerrelayprovider"":false,""__relay_internal__pv__BarcelonaHasDisplayNamesrelayprovider"":false,""__relay_internal__pv__BarcelonaCanSeeSponsoredContentrelayprovider"":false,""__relay_internal__pv__BarcelonaShouldShowFediverseM075Featuresrelayprovider"":true,""__relay_internal__pv__BarcelonaShouldShowTagRedesignrelayprovider"":false,""__relay_internal__pv__BarcelonaIsInternalUserrelayprovider"":false"
|
||||
Try
|
||||
DownloadCheckCredentials()
|
||||
|
||||
Responser.Method = "POST"
|
||||
Responser.Referer = "https://www.threads.net/"
|
||||
Responser.Headers.Add(GQL_HEADER_FB_LSD, Token_lsd)
|
||||
Responser.Headers.Add(GQL_HEADER_FB_FRINDLY_NAME, If(Cursor.IsEmptyString, GQL_S_NAME_1, GQL_S_NAME_2))
|
||||
With Responser
|
||||
.Method = "POST"
|
||||
.Referer = "https://www.threads.net/"
|
||||
.ContentType = "application/x-www-form-urlencoded"
|
||||
With .Headers
|
||||
.Add(GQL_HEADER_FB_LSD, Token_lsd)
|
||||
'.Add(GQL_HEADER_FB_FRINDLY_NAME, If(Cursor.IsEmptyString, GQL_S_NAME_1, GQL_S_NAME_2))
|
||||
.Add(GQL_HEADER_FB_FRINDLY_NAME, GQL_S_NAME_2)
|
||||
End With
|
||||
End With
|
||||
|
||||
Dim nextCursor$ = String.Empty
|
||||
Dim dataFound As Boolean = False
|
||||
|
||||
Dim vars$ = SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & If(Cursor.IsEmptyString, var_init, String.Format(var_cursor, Cursor)) & "}")
|
||||
'Dim vars$ = SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & If(Cursor.IsEmptyString, var_init, String.Format(var_cursor, Cursor)) & "}")
|
||||
Dim vars$ = String.Format(PayloadData, Token_lsd, Token_dtsg_Var, GQL_S_DOC_ID_2, GQL_S_NAME_2,
|
||||
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(var_cursor2, IIf(Cursor.IsEmptyString, "null", $"""{Cursor}""")) & "}"))
|
||||
|
||||
If Cursor.IsEmptyString Then
|
||||
URL = String.Format(GQL_Q, Token_lsd, Token_dtsg_Var, GQL_S_DOC_ID_1, GQL_S_NAME_1, vars)
|
||||
Else
|
||||
URL = String.Format(GQL_Q, Token_lsd, Token_dtsg_Var, GQL_S_DOC_ID_2, GQL_S_NAME_2, vars)
|
||||
End If
|
||||
'If Cursor.IsEmptyString Then
|
||||
' URL = String.Format(GQL_Q, Token_lsd, Token_dtsg_Var, GQL_S_DOC_ID_1, GQL_S_NAME_1, vars)
|
||||
'Else
|
||||
' URL = String.Format(GQL_Q, Token_lsd, Token_dtsg_Var, GQL_S_DOC_ID_2, GQL_S_NAME_2, vars)
|
||||
'End If
|
||||
|
||||
Using j As EContainer = GetDocument(URL, Token)
|
||||
If j.ListExists Then
|
||||
With j({"data", "xdt_viewer", "text_app_saved_media"})
|
||||
If .ListExists Then
|
||||
nextCursor = .Value({"page_info"}, "end_cursor")
|
||||
With .Item({"edges"})
|
||||
If .ListExists Then dataFound = DefaultParser(.Self, Sections.Timeline, Token)
|
||||
End With
|
||||
End If
|
||||
End With
|
||||
Using j As EContainer = GetDocument(GQL_Q2, vars, Token)
|
||||
If Not CheckErrors(j) Then
|
||||
If j.ListExists Then
|
||||
With j({"data", "xdt_text_app_viewer", "saved_media"})
|
||||
If .ListExists Then
|
||||
nextCursor = .Value({"page_info"}, "end_cursor")
|
||||
With .Item({"edges"})
|
||||
If .ListExists Then dataFound = DefaultParser(.Self, Sections.Timeline, Token)
|
||||
End With
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
Else
|
||||
Throw New JsonErrorException(JsonErrorMessage(j))
|
||||
End If
|
||||
End Using
|
||||
|
||||
If dataFound And Not nextCursor.IsEmptyString Then DownloadSavedPosts(nextCursor, Token)
|
||||
If dataFound And Not nextCursor.IsEmptyString Then DownloadSavedPosts(nextCursor, 0, Token)
|
||||
Catch uex As JsonErrorException
|
||||
If Round > 0 Then
|
||||
ProcessJsonErrorException(uex)
|
||||
ElseIf Not _IdChanged AndAlso UpdateCredentials() Then
|
||||
DownloadSavedPosts(Cursor, Round + 1, Token)
|
||||
Else
|
||||
ProcessJsonErrorException(uex)
|
||||
End If
|
||||
Catch eex As ExitException
|
||||
Throw eex
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, $"saved posts downloading error [{URL}]")
|
||||
ProcessException(ex, Token, "saved posts downloading error")
|
||||
End Try
|
||||
End Sub
|
||||
Private Function GetDocument(ByVal URL As String, ByVal Token As CancellationToken, Optional ByVal Round As Integer = 0) As EContainer
|
||||
Private Function GetDocument(ByVal URL As String, ByVal PayLoad As String, ByVal Token As CancellationToken, Optional ByVal Round As Integer = 0) As EContainer
|
||||
Try
|
||||
ThrowAny(Token)
|
||||
If Round > 0 AndAlso Not UpdateCredentials() Then DisableDownload() : Throw New Exception("Failed to update credentials")
|
||||
ThrowAny(Token)
|
||||
WaitTimer()
|
||||
Dim r$ = Responser.GetResponse(URL)
|
||||
Dim r$ = Responser.GetResponse(URL, PayLoad)
|
||||
If Not r.IsEmptyString Then Return JsonDocument.Parse(r) Else Throw New Exception("Failed to get a response")
|
||||
Catch ex As Exception
|
||||
If Round = 0 Then
|
||||
Return GetDocument(URL, Token, Round + 1)
|
||||
Return GetDocument(URL, PayLoad, Token, Round + 1)
|
||||
Else
|
||||
Throw ex
|
||||
End If
|
||||
End Try
|
||||
End Function
|
||||
Private _IdChanged As Boolean = False
|
||||
Private Function UpdateCredentials(Optional ByVal e As ErrorsDescriber = Nothing) As Boolean
|
||||
Dim URL$ = If(IsSavedPosts, "https://www.threads.net/", $"https://www.threads.net/@{NameTrue}")
|
||||
ResetBaseTokens()
|
||||
@@ -294,9 +382,25 @@ Namespace API.ThreadsNet
|
||||
End With
|
||||
WaitTimer()
|
||||
Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException)
|
||||
Dim newID$
|
||||
Dim idStr$ = String.Empty
|
||||
If Not r.IsEmptyString Then
|
||||
ParseTokens(r, 0)
|
||||
If ID.IsEmptyString Then ID = RegexReplace(r, RParams.DMS("""props"":\{""user_id"":""(\d+)""", 1, EDP.ReturnValue))
|
||||
newID = RegexReplace(r, RParams.DMS("""props"":\{[^\{\}]*?""user_id"":""(\d+)""", 1, EDP.ReturnValue))
|
||||
If ID.IsEmptyString OrElse ID = newID Then
|
||||
_IdChanged = ID.IsEmptyString
|
||||
ID = newID
|
||||
Else
|
||||
_IdChanged = True
|
||||
idStr = $"user ID changed from {ID} to {newID}"
|
||||
LogError(Nothing, idStr)
|
||||
ID = newID
|
||||
End If
|
||||
If _IdChanged Then
|
||||
If Not idStr.IsEmptyString Then UserDescriptionUpdate(idStr, True, True, True)
|
||||
_ForceSaveUserInfo = True
|
||||
_ForceSaveUserInfoOnException = True
|
||||
End If
|
||||
End If
|
||||
Return Valid
|
||||
Catch ex As Exception
|
||||
@@ -322,20 +426,22 @@ Namespace API.ThreadsNet
|
||||
#End Region
|
||||
#Region "ReparseMissing"
|
||||
Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken)
|
||||
Const varsPattern$ = """postID"":""{0}"",""userID"":""{1}"",""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsFeedbackHubEnabledrelayprovider"":false"
|
||||
'Const varsPattern$ = """postID"":""{0}"",""userID"":""{1}"",""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsFeedbackHubEnabledrelayprovider"":false"
|
||||
Const varsPattern$ = """postID"":""{0}"",""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaShouldShowFediverseM1Featuresrelayprovider"":true,""__relay_internal__pv__BarcelonaShareableListsrelayprovider"":true,""__relay_internal__pv__BarcelonaIsSearchDiscoveryEnabledrelayprovider"":false,""__relay_internal__pv__BarcelonaOptionalCookiesEnabledrelayprovider"":true,""__relay_internal__pv__BarcelonaQuotedPostUFIEnabledrelayprovider"":false,""__relay_internal__pv__BarcelonaIsCrawlerrelayprovider"":false,""__relay_internal__pv__BarcelonaHasDisplayNamesrelayprovider"":false,""__relay_internal__pv__BarcelonaCanSeeSponsoredContentrelayprovider"":false,""__relay_internal__pv__BarcelonaShouldShowFediverseM075Featuresrelayprovider"":true,""__relay_internal__pv__BarcelonaShouldShowTagRedesignrelayprovider"":false,""__relay_internal__pv__BarcelonaIsInternalUserrelayprovider"":false,""__relay_internal__pv__BarcelonaInlineComposerEnabledrelayprovider"":false"
|
||||
'Const varsPattern$ = "{""postID"":""{0}"",""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsFeedbackHubEnabledrelayprovider"":false}"
|
||||
Const urlPattern$ = "https://www.threads.net/api/graphql?lsd={0}&variables={1}&fb_api_req_friendly_name=BarcelonaPostPageQuery&server_timestamps=true&fb_dtsg={2}&doc_id=25460088156920903"
|
||||
'Const urlPattern$ = "https://www.threads.net/api/graphql?lsd={0}&variables={1}&fb_api_req_friendly_name=BarcelonaPostPageQuery&server_timestamps=true&fb_dtsg={2}&doc_id=25460088156920903"
|
||||
Dim rList As New List(Of Integer)
|
||||
Dim URL$ = String.Empty
|
||||
DefaultParser_ElemNode = Nothing
|
||||
DefaultParser_IgnorePass = True
|
||||
Try
|
||||
If ContentMissingExists Then
|
||||
Responser.Method = "POST"
|
||||
Responser.ContentType = "application/x-www-form-urlencoded"
|
||||
Responser.Referer = $"https://www.threads.net/@{NameTrue}"
|
||||
If Not IsSingleObjectDownload AndAlso Not UpdateCredentials() Then Throw New Exception("Failed to update credentials")
|
||||
Dim m As UserMedia
|
||||
Dim vars$
|
||||
Dim r As Byte
|
||||
Dim j As EContainer
|
||||
ProgressPre.ChangeMax(_ContentList.Count)
|
||||
For i% = 0 To _ContentList.Count - 1
|
||||
@@ -343,21 +449,39 @@ Namespace API.ThreadsNet
|
||||
m = _ContentList(i)
|
||||
If m.State = UserMedia.States.Missing And Not m.Post.ID.IsEmptyString Then
|
||||
ThrowAny(Token)
|
||||
vars = SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(varsPattern, m.Post.ID.Split("_").FirstOrDefault, ID) & "}")
|
||||
URL = String.Format(urlPattern, Token_lsd, vars, Token_dtsg_Var)
|
||||
'vars = SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(varsPattern, m.Post.ID.Split("_").FirstOrDefault, ID) & "}")
|
||||
'URL = String.Format(urlPattern, Token_lsd, vars, Token_dtsg_Var)
|
||||
|
||||
j = GetDocument(URL, Token)
|
||||
If j.ListExists Then
|
||||
With j.ItemF({"data", "data", "edges", 0, "node", "thread_items", 0, "post"})
|
||||
If .ListExists AndAlso DefaultParser({ .Self}, Sections.Timeline, Token) Then rList.Add(i)
|
||||
End With
|
||||
j.Dispose()
|
||||
End If
|
||||
vars = String.Format(PayloadData, Token_lsd, Token_dtsg_Var, "9094233770675261", "BarcelonaPostPageDirectQuery",
|
||||
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(varsPattern, m.Post.ID) & "}"))
|
||||
|
||||
For r = 0 To 1
|
||||
j = GetDocument(GQL_Q2, vars, Token)
|
||||
If Not CheckErrors(j) Then
|
||||
If j.ListExists Then
|
||||
With j.ItemF({"data", "data", "edges", 0, "node", "thread_items", 0, "post"})
|
||||
If .ListExists AndAlso DefaultParser({ .Self}, Sections.Timeline, Token) Then rList.Add(i)
|
||||
End With
|
||||
j.Dispose()
|
||||
End If
|
||||
Else
|
||||
j.DisposeIfReady(False)
|
||||
If r > 0 Then
|
||||
ProcessJsonErrorException(New JsonErrorException(JsonErrorMessage(j)))
|
||||
ElseIf Not _IdChanged AndAlso UpdateCredentials() Then
|
||||
Continue For
|
||||
Else
|
||||
ProcessJsonErrorException(New JsonErrorException(JsonErrorMessage(j)))
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
Catch eex As ExitException
|
||||
Throw eex
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, $"ReparseMissing error [{URL}]")
|
||||
ProcessException(ex, Token, "reparseMissing error")
|
||||
Finally
|
||||
DefaultParser_ElemNode = DefaultParser_ElemNode_Default
|
||||
DefaultParser_IgnorePass = False
|
||||
@@ -375,9 +499,9 @@ Namespace API.ThreadsNet
|
||||
If Not postCode.IsEmptyString Then
|
||||
Dim postId$ = CodeToID(postCode)
|
||||
If Not postId.IsEmptyString Then
|
||||
_NameTrue = MySettings.IsMyUser(url).UserName
|
||||
NameTrue = MySettings.IsMyUser(url).UserName
|
||||
DefaultParser_PostUrlCreator = Function(post) url
|
||||
If Not _NameTrue.IsEmptyString AndAlso UpdateCredentials(EDP.ReturnValue) Then
|
||||
If Not NameTrue(True).IsEmptyString AndAlso UpdateCredentials(EDP.ReturnValue) Then
|
||||
_ContentList.Add(New UserMedia(url) With {.State = UserMedia.States.Missing, .Post = postId})
|
||||
ReparseMissing(Token)
|
||||
End If
|
||||
|
||||
Reference in New Issue
Block a user