From 496c9487cd3778c9d8942d74b4340d5cf96ef723 Mon Sep 17 00:00:00 2001
From: Andy <88590076+AAndyProgram@users.noreply.github.com>
Date: Wed, 15 Nov 2023 23:50:34 +0300
Subject: [PATCH] 2023.11.15.0
ADD FACEBOOK
SiteSettingsBase: update 'CLONE_PROPERTIES' function (exclude 'DoNotUse' attribute)
API.Instagram: handle 401 error
API.ThreadsNet.SiteSettings: make the class compatible for Facebook
xHanster, XVideos, PornHub, ThisVid: update download function for search queries
Hosts.PropertyValueHost: set the 'Exists' value based on the 'DoNotUse' attribute
Hosts.SettingsHost: use 'GetObjectMembers' instead of 'GetTypeInfo.DeclaredMembers' to get class members
---
.../Attributes/Attributes.vb | 6 +
SCrawler/API/Base/DeclaredNames.vb | 2 +
SCrawler/API/Base/SiteSettingsBase.vb | 10 +-
SCrawler/API/Facebook/Declarations.vb | 37 +
SCrawler/API/Facebook/SiteSettings.vb | 110 +++
SCrawler/API/Facebook/UserData.vb | 713 ++++++++++++++++++
SCrawler/API/Facebook/UserExchangeOptions.vb | 32 +
SCrawler/API/Instagram/UserData.vb | 8 +-
SCrawler/API/PornHub/UserData.vb | 46 +-
SCrawler/API/ThisVid/UserData.vb | 15 +-
SCrawler/API/ThreadsNet/SiteSettings.vb | 66 +-
SCrawler/API/ThreadsNet/UserData.vb | 2 +-
SCrawler/API/XVIDEOS/UserData.vb | 56 +-
SCrawler/API/Xhamster/UserData.vb | 39 +-
.../Icons/SiteIcons/FacebookIcon_32.ico | Bin 0 -> 5430 bytes
.../Pictures/SitePictures/FacebookPic_37.png | Bin 0 -> 1370 bytes
.../PluginsEnvironment/Hosts/PluginHost.vb | 1 +
.../Hosts/PropertyValueHost.vb | 2 +-
.../PluginsEnvironment/Hosts/SettingsHost.vb | 4 +-
SCrawler/SCrawler.vbproj | 10 +
SCrawler/SiteResources.Designer.vb | 20 +
SCrawler/SiteResources.resx | 6 +
22 files changed, 1125 insertions(+), 60 deletions(-)
create mode 100644 SCrawler/API/Facebook/Declarations.vb
create mode 100644 SCrawler/API/Facebook/SiteSettings.vb
create mode 100644 SCrawler/API/Facebook/UserData.vb
create mode 100644 SCrawler/API/Facebook/UserExchangeOptions.vb
create mode 100644 SCrawler/Content/Icons/SiteIcons/FacebookIcon_32.ico
create mode 100644 SCrawler/Content/Pictures/SitePictures/FacebookPic_37.png
diff --git a/SCrawler.PluginProvider/Attributes/Attributes.vb b/SCrawler.PluginProvider/Attributes/Attributes.vb
index 371d5e2..96e8028 100644
--- a/SCrawler.PluginProvider/Attributes/Attributes.vb
+++ b/SCrawler.PluginProvider/Attributes/Attributes.vb
@@ -65,6 +65,12 @@ Namespace Plugin.Attributes
End Class
''' Attribute to disable some properties for host use
Public NotInheritable Class DoNotUse : Inherits Attribute
+ Public ReadOnly Value As Boolean = True
+ Public Sub New()
+ End Sub
+ Public Sub New(ByVal Value As Boolean)
+ Me.Value = Value
+ End Sub
End Class
''' Special property updater
Public NotInheritable Class PropertyUpdater : Inherits Attribute
diff --git a/SCrawler/API/Base/DeclaredNames.vb b/SCrawler/API/Base/DeclaredNames.vb
index 2076ab4..0991bb7 100644
--- a/SCrawler/API/Base/DeclaredNames.vb
+++ b/SCrawler/API/Base/DeclaredNames.vb
@@ -11,6 +11,8 @@ Namespace API.Base
Friend Const Header_Authorization As String = "authorization"
Friend Const Header_CSRFToken As String = "x-csrf-token"
+ Friend Const Header_FB_FRIENDLY_NAME As String = "x-fb-friendly-name"
+
Friend Const ConcurrentDownloadsCaption As String = "Concurrent downloads"
Friend Const ConcurrentDownloadsToolTip As String = "The number of concurrent downloads."
Friend Const SavedPostsUserNameCaption As String = "Saved posts user"
diff --git a/SCrawler/API/Base/SiteSettingsBase.vb b/SCrawler/API/Base/SiteSettingsBase.vb
index e9e223c..e36085c 100644
--- a/SCrawler/API/Base/SiteSettingsBase.vb
+++ b/SCrawler/API/Base/SiteSettingsBase.vb
@@ -278,9 +278,13 @@ Namespace API.Base
'1 = clone
'2 = any
Dim filterUC As Func(Of MemberInfo, Byte, Boolean) = Function(ByVal m As MemberInfo, ByVal __mode As Byte) As Boolean
- With m.GetCustomAttribute(Of PClonableAttribute)
- Return Not .Self Is Nothing AndAlso (__mode = 2 OrElse If(__mode = 0, .Update, .Clone))
- End With
+ If m.GetCustomAttribute(Of DoNotUse) Is Nothing Then
+ Return False
+ Else
+ With m.GetCustomAttribute(Of PClonableAttribute)
+ Return Not .Self Is Nothing AndAlso (__mode = 2 OrElse If(__mode = 0, .Update, .Clone))
+ End With
+ End If
End Function
Dim filterAll As Func(Of MemberInfo, Boolean) = Function(m) filterUC.Invoke(m, 2)
Dim filterC As Func(Of MemberInfo, Boolean) = Function(m) If(Full, filterAll.Invoke(m), filterUC.Invoke(m, 1))
diff --git a/SCrawler/API/Facebook/Declarations.vb b/SCrawler/API/Facebook/Declarations.vb
new file mode 100644
index 0000000..de6d84e
--- /dev/null
+++ b/SCrawler/API/Facebook/Declarations.vb
@@ -0,0 +1,37 @@
+' Copyright (C) 2023 Andy https://github.com/AAndyProgram
+' This program is free software: you can redistribute it and/or modify
+' it under the terms of the GNU General Public License as published by
+' the Free Software Foundation, either version 3 of the License, or
+' (at your option) any later version.
+'
+' This program is distributed in the hope that it will be useful,
+' but WITHOUT ANY WARRANTY
+Imports System.Text.RegularExpressions
+Imports PersonalUtilities.Functions.XML.Base
+Imports PersonalUtilities.Functions.RegularExpressions
+Namespace API.Facebook
+ Friend Module Declarations
+ Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
+ Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
+ Friend ReadOnly Regex_UserID As RParams = RParams.DMS("userid.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue)
+
+ Friend ReadOnly Regex_Photos_by As RParams = RParams.DMS("photos_by"",""id"":""([^""]+)", 1, EDP.ReturnValue)
+ Friend ReadOnly Regex_FileName As RParams = RParams.DM("([^/\?]+\..{3,4})(?=(\?|\Z))", 0, EDP.ReturnValue)
+ Friend ReadOnly Regex_ProfileUrlID As RParams = RParams.DMS("profile.php\?id=(\d+)", 1, EDP.ReturnValue)
+ Friend ReadOnly Regex_VideoPageID As RParams = RParams.DMS("pageid.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue)
+ Friend ReadOnly Regex_StoryBucket As RParams = RParams.DMS("story_bucket[^\>]*?(\d+)", 1, EDP.ReturnValue)
+
+ Friend ReadOnly Regex_VideoIDFromURL As RParams = RParams.DMS("facebook.com/([^/]+/videos/|watch/\D*[\?&]{1}v=)(\d+)", 2, EDP.ReturnValue)
+ Friend ReadOnly Regex_PostHtmlFullPicture As RParams = RParams.DM("^((?!_[ps]{1}\d+x\d+).)*$", 0, EDP.ReturnValue)
+
+ Friend ReadOnly SpecialNode() As NodeParams = {New NodeParams("attachment", True, True, True, True, 30),
+ New NodeParams("media", True, True, True, True, 0),
+ New NodeParams("photo_image", True, True, True, True, 0),
+ New NodeParams("uri", True, True, True, True, 0)}
+ Friend ReadOnly SpecialNode2() As NodeParams = {New NodeParams("result", True, True, True, True, 30),
+ New NodeParams("data", True, True, True, True, 0),
+ New NodeParams("currmedia", True, True, True, True, 0),
+ New NodeParams("image", True, True, True, True, 0),
+ New NodeParams("uri", True, True, True, True, 0)}
+ End Module
+End Namespace
\ No newline at end of file
diff --git a/SCrawler/API/Facebook/SiteSettings.vb b/SCrawler/API/Facebook/SiteSettings.vb
new file mode 100644
index 0000000..6ff2b34
--- /dev/null
+++ b/SCrawler/API/Facebook/SiteSettings.vb
@@ -0,0 +1,110 @@
+' Copyright (C) 2023 Andy https://github.com/AAndyProgram
+' This program is free software: you can redistribute it and/or modify
+' it under the terms of the GNU General Public License as published by
+' the Free Software Foundation, either version 3 of the License, or
+' (at your option) any later version.
+'
+' This program is distributed in the hope that it will be useful,
+' but WITHOUT ANY WARRANTY
+Imports SCrawler.API.Base
+Imports SCrawler.Plugin
+Imports SCrawler.Plugin.Attributes
+Imports PersonalUtilities.Tools.Web.Clients
+Imports PersonalUtilities.Functions.RegularExpressions
+Namespace API.Facebook
+
+ Friend Class SiteSettings : Inherits ThreadsNet.SiteSettings
+#Region "Declarations"
+#Region "Auth"
+
+ Friend ReadOnly Property Header_Accept As PropertyValue
+
+ Friend Overrides ReadOnly Property HH_IG_APP_ID As PropertyValue
+ Get
+ Return __HH_IG_APP_ID
+ End Get
+ End Property
+ Friend Overrides ReadOnly Property HH_CSRF_TOKEN As PropertyValue
+ Get
+ Return __HH_CSRF_TOKEN
+ End Get
+ End Property
+
+ Friend ReadOnly Property HH_PLATFORM_VER As PropertyValue
+#End Region
+#Region "Defaults"
+
+ Friend ReadOnly Property ParsePhotoBlock As PropertyValue
+
+ Friend ReadOnly Property ParseVideoBlock As PropertyValue
+
+ Friend ReadOnly Property ParseStoriesBlock As PropertyValue
+#End Region
+#End Region
+#Region "Initializer"
+ Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
+ MyBase.New("Facebook", "facebook.com", AccName, Temp, My.Resources.SiteResources.FacebookIcon_32, My.Resources.SiteResources.FacebookPic_37)
+
+ With Responser.Headers
+ .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Authority, "www.facebook.com"))
+ .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Origin, "https://www.facebook.com"))
+ .Remove(DeclaredNames.Header_FB_FRIENDLY_NAME)
+ End With
+ Header_Accept = New PropertyValue(String.Empty, GetType(String))
+ HH_PLATFORM_VER = New PropertyValue(String.Empty, GetType(String))
+ ParsePhotoBlock = New PropertyValue(True)
+ ParseVideoBlock = New PropertyValue(True)
+ ParseStoriesBlock = New PropertyValue(True)
+
+ UrlPatternUser = "https://www.facebook.com/{0}"
+ UserRegex = RParams.DMS("facebook.com/(profile.php\?id=\d+|[^\?&/]+)", 1)
+ ImageVideoContains = "facebook.com"
+ UserOptionsType = GetType(UserExchangeOptions)
+ End Sub
+#End Region
+#Region "GetInstance"
+ Friend Overrides Function GetInstance(ByVal What As ISiteSettings.Download) As IPluginContentProvider
+ Return New UserData
+ End Function
+#End Region
+#Region "UpdateResponserData"
+ Friend Overrides Sub UpdateResponserData(ByVal Resp As Responser)
+ With Responser.Cookies
+ .Update(Resp.Cookies)
+ If .Changed Then Responser.SaveCookies() : .Changed = False
+ End With
+ End Sub
+#End Region
+#Region "BaseAuthExists, GetUserUrl, GetUserPostUrl, IsMyUser, IsMyImageVideo"
+ Friend Overrides Function BaseAuthExists() As Boolean
+ Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value)
+ End Function
+ Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String
+ Return DirectCast(User, UserData).GetProfileUrl
+ End Function
+ Friend Overrides Function GetUserPostUrl(ByVal User As UserDataBase, ByVal Media As UserMedia) As String
+ Return Media.URL_BASE
+ End Function
+ Friend Overrides Function IsMyUser(ByVal UserURL As String) As ExchangeOptions
+ Dim e As ExchangeOptions = MyBase.IsMyUser(UserURL)
+ If e.Exists Then
+ e.Options = e.UserName
+ Dim v$ = RegexReplace(e.UserName, Regex_ProfileUrlID)
+ If Not v.IsEmptyString Then
+ e.UserName = v
+ Else
+ e.UserName = e.UserName.StringRemoveWinForbiddenSymbols
+ End If
+ End If
+ Return e
+ End Function
+ Friend Overrides Function IsMyImageVideo(ByVal URL As String) As ExchangeOptions
+ If Not URL.IsEmptyString AndAlso Not CStr(AConvert(Of String)(URL, Regex_VideoIDFromURL, String.Empty)).IsEmptyString Then
+ Return New ExchangeOptions(Site, String.Empty) With {.Exists = True}
+ Else
+ Return Nothing
+ End If
+ End Function
+#End Region
+ End Class
+End Namespace
\ No newline at end of file
diff --git a/SCrawler/API/Facebook/UserData.vb b/SCrawler/API/Facebook/UserData.vb
new file mode 100644
index 0000000..1fce06c
--- /dev/null
+++ b/SCrawler/API/Facebook/UserData.vb
@@ -0,0 +1,713 @@
+' Copyright (C) 2023 Andy https://github.com/AAndyProgram
+' This program is free software: you can redistribute it and/or modify
+' it under the terms of the GNU General Public License as published by
+' the Free Software Foundation, either version 3 of the License, or
+' (at your option) any later version.
+'
+' This program is distributed in the hope that it will be useful,
+' but WITHOUT ANY WARRANTY
+Imports System.Threading
+Imports System.Text.RegularExpressions
+Imports SCrawler.API.Base
+Imports SCrawler.API.YouTube.Objects
+Imports PersonalUtilities.Functions.XML
+Imports PersonalUtilities.Functions.RegularExpressions
+Imports PersonalUtilities.Tools.Web.Clients
+Imports PersonalUtilities.Tools.Web.Documents.JSON
+Imports IG = SCrawler.API.Instagram.SiteSettings
+Imports UTypes = SCrawler.API.Base.UserMedia.Types
+Imports UStates = SCrawler.API.Base.UserMedia.States
+Namespace API.Facebook
+ Friend Class UserData : Inherits Instagram.UserData
+#Region "XML names"
+ Private Const Name_IsNoNameProfile As String = "IsNoNameProfile"
+ Private Const Name_OptionsParsed As String = "OptionsParsed"
+ Private Const Name_VideoPageID As String = "VideoPageID"
+ Private Const Name_StoryBucket As String = "StoryBucket"
+ Private Const Name_ParsePhotoBlock As String = "ParsePhotoBlock"
+ Private Const Name_ParseVideoBlock As String = "ParseVideoBlock"
+ Private Const Name_ParseStoriesBlock As String = "ParseStoriesBlock"
+#End Region
+#Region "Declarations"
+ Friend ReadOnly Property MySettings As SiteSettings
+ Get
+ Return HOST.Source
+ End Get
+ End Property
+ Private IsNoNameProfile As Boolean = False
+ Private OptionsParsed As Boolean = False
+ Private Property VideoPageID As String = String.Empty
+ Private Property StoryBucket As String = String.Empty
+ Friend Property ParsePhotoBlock As Boolean = True
+ Friend Property ParseVideoBlock As Boolean = True
+ Friend Property ParseStoriesBlock As Boolean = True
+ Private Enum PageBlock As Integer
+ Timeline = Sections.Timeline
+ Stories = Sections.Stories
+ Photos = 100
+ Videos = 101
+ Undefined = -1
+ End Enum
+#End Region
+#Region "GetProfileUrl"
+ Friend Function GetProfileUrl() As String
+ If IsNoNameProfile Then
+ Return $"https://www.facebook.com/profile.php?id={ID}"
+ Else
+ Return $"https://www.facebook.com/{NameTrue}"
+ End If
+ End Function
+#End Region
+#Region "Exchange"
+ Friend Overrides Function ExchangeOptionsGet() As Object
+ Return New UserExchangeOptions(Me)
+ End Function
+ Friend Overrides Sub ExchangeOptionsSet(ByVal Obj As Object)
+ If Not Obj Is Nothing AndAlso TypeOf Obj Is UserExchangeOptions Then
+ With DirectCast(Obj, UserExchangeOptions)
+ ParsePhotoBlock = .ParsePhotoBlock
+ ParseVideoBlock = .ParseVideoBlock
+ ParseStoriesBlock = .ParseStoriesBlock
+ End With
+ End If
+ End Sub
+#End Region
+#Region "Loader"
+ Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)
+ Dim updateNames As Action = Sub()
+ If Not OptionsParsed AndAlso Not Options.IsEmptyString Then
+ OptionsParsed = True
+ Dim v$ = RegexReplace(Options, Regex_ProfileUrlID)
+ If Not v.IsEmptyString Then ID = v : IsNoNameProfile = True
+ End If
+ End Sub
+ With Container
+ If Loading Then
+ If .Contains(Name_IsNoNameProfile) Then
+ IsNoNameProfile = .Value(Name_IsNoNameProfile).FromXML(Of Boolean)(False)
+ Else
+ updateNames.Invoke
+ End If
+ OptionsParsed = .Value(Name_OptionsParsed).FromXML(Of Boolean)(False)
+ VideoPageID = .Value(Name_VideoPageID)
+ StoryBucket = .Value(Name_StoryBucket)
+ ParsePhotoBlock = .Value(Name_ParsePhotoBlock).FromXML(Of Boolean)(True)
+ ParseVideoBlock = .Value(Name_ParseVideoBlock).FromXML(Of Boolean)(True)
+ ParseStoriesBlock = .Value(Name_ParseStoriesBlock).FromXML(Of Boolean)(True)
+ Else
+ updateNames.Invoke
+ .Add(Name_IsNoNameProfile, IsNoNameProfile.BoolToInteger)
+ .Add(Name_OptionsParsed, OptionsParsed.BoolToInteger)
+ .Add(Name_VideoPageID, VideoPageID)
+ .Add(Name_StoryBucket, StoryBucket)
+ .Add(Name_ParsePhotoBlock, ParsePhotoBlock.BoolToInteger)
+ .Add(Name_ParseVideoBlock, ParseVideoBlock.BoolToInteger)
+ .Add(Name_ParseStoriesBlock, ParseStoriesBlock.BoolToInteger)
+ End If
+ End With
+ End Sub
+#End Region
+#Region "Download functions"
+ Private Token_dtsg As String = String.Empty
+ Private Token_lsd As String = String.Empty
+ Private Token_Photosby As String = String.Empty
+ Private Limit As Integer = -1
+ Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
+ Try
+ GetUserTokens(Token)
+ LoadSavePostsKV(True)
+ Limit = If(DownloadTopCount, -1)
+ If IsSavedPosts Then
+ DownloadData_SavedPosts(String.Empty, Token)
+ Else
+ If DownloadImages And ParsePhotoBlock Then DownloadData_Photo(String.Empty, Token)
+ If DownloadVideos And ParseVideoBlock Then DownloadData_Video(String.Empty, Token)
+ If (DownloadImages Or DownloadVideos) And ParseStoriesBlock Then DownloadData_Stories(Token)
+ End If
+ LoadSavePostsKV(False)
+ Finally
+ MySettings.UpdateResponserData(Responser)
+ End Try
+ End Sub
+ Private Const Header_fb_fr_name_Photo As String = "ProfileCometAppCollectionPhotosRendererPaginationQuery"
+ Private Const Header_fb_fr_name_Video As String = "PagesCometChannelTabAllVideosCardImplPaginationQuery"
+ Private Const Header_fb_fr_name_Stories As String = "StoriesSuspenseContentPaneRootWithEntryPointQuery"
+ Private Const Header_fb_fr_name_SavedPosts As String = "CometSaveDashboardAllItemsPaginationQuery"
+ Private Const DocID_Photo As String = "6684543058255697"
+ Private Const DocID_Video As String = "24545934291687581"
+ Private Const DocID_Stories As String = "6771064226315961"
+ Private Const DocID_SavedPosts As String = "7112228098805003"
+ Private Const Graphql_UrlPattern As String = "https://www.facebook.com/api/graphql?lsd={0}&doc_id={1}&server_timestamps=true&fb_dtsg={3}&fb_api_req_friendly_name={2}&variables={4}"
+ Private Const VideoHtmlUrlPattern As String = "https://www.facebook.com/watch/?v={0}"
+ Private Sub DownloadData_Photo(ByVal Cursor As String, ByVal Token As CancellationToken)
+ Dim URL$ = String.Empty
+ Const VarPattern$ = """count"":8,""cursor"":""{0}"",""scale"":1,""id"":""{1}"""
+ Try
+ Dim nextCursor$ = String.Empty
+ Dim newPostsDetected As Boolean = False
+ Dim pUrl$, pUrlBase$
+ Dim pid As PostKV
+
+ ValidateBaseTokens()
+ If Token_Photosby.IsEmptyString Then Throw New ArgumentNullException("Token_Photosby", "Unable to obtain token")
+
+ URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Photo, Header_fb_fr_name_Photo,
+ SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg),
+ SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, Cursor, Token_Photosby) & "}"))
+
+ ResponserApplyDefs(Header_fb_fr_name_Photo)
+ ThrowAny(Token)
+
+ Dim r$ = Responser.GetResponse(URL)
+ If Not r.IsEmptyString Then
+ Using j As EContainer = JsonDocument.Parse(r)
+ If j.ListExists Then
+ With j({"data", "node", "pageItems", "edges"})
+ If .ListExists Then
+ ProgressPre.ChangeMax(.Count)
+ For Each jNode As EContainer In .Self
+ ProgressPre.Perform()
+ With jNode
+ If Not .Value("cursor").IsEmptyString Then nextCursor = .Value("cursor")
+ With .Item({"node"})
+ If .ListExists Then
+ pUrl = .Value({"node", "viewer_image"}, "uri")
+ pUrlBase = .Value("url")
+ If Not pUrl.IsEmptyString Then
+ pid = New PostKV(.Value("id"), .Value({"node"}, "id"), PageBlock.Photos)
+ If Not PostKvExists(pid) Then
+ newPostsDetected = True
+ PostsKVIDs.ListAddValue(pid, LNC)
+ _TempPostsList.Add(pid.ID)
+ _TempMediaList.ListAddValue(New UserMedia(pUrl, UTypes.Picture) With {
+ .URL_BASE = pUrlBase,
+ .File = CreateFileFromUrl(pUrl),
+ .Post = pid.ID.IfNullOrEmpty(pid.Code)}, LNC)
+ If Limit > 0 And _TempMediaList.Count >= Limit Then Exit Sub
+ Else
+ Exit Sub
+ End If
+ End If
+ End If
+ End With
+ End With
+ Next
+ End If
+ End With
+ End If
+ End Using
+ End If
+
+ If newPostsDetected And Not nextCursor.IsEmptyString Then DownloadData_Photo(nextCursor, Token)
+ Catch ex As Exception
+ ProcessException(ex, Token, $"data (photo) downloading error [{URL}]",, Responser)
+ End Try
+ End Sub
+ Private Sub DownloadData_Video(ByVal Cursor As String, ByVal Token As CancellationToken)
+ Dim URL$ = String.Empty
+ Const VarPattern$ = """alwaysIncludeAudioRooms"":true,""count"":6,""cursor"":{0},""pageID"":""{1}"",""scale"":4,""showReactions"":true,""useDefaultActor"":false,""id"":""{1}"""
+ Try
+ Dim nextCursor$ = String.Empty
+ Dim newPostsDetected As Boolean = False
+ Dim pid As PostKV
+
+ If VideoPageID.IsEmptyString Then GetVideoPageID(Token)
+ If VideoPageID.IsEmptyString Then Throw New ArgumentNullException("VideoPageID", "Unable to obtain VideoPageID")
+ ValidateBaseTokens()
+
+ URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Video, Header_fb_fr_name_Video,
+ SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg),
+ SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, If(Cursor.IsEmptyString, "null", $"""{Cursor}"""), VideoPageID) & "}"))
+
+ ResponserApplyDefs(Header_fb_fr_name_Video)
+ ThrowAny(Token)
+
+ Dim r$ = Responser.GetResponse(URL)
+ If Not r.IsEmptyString Then
+ Using j As EContainer = JsonDocument.Parse(r)
+ If j.ListExists Then
+ With j({"data", "node", "all_videos", "edges"})
+ If .ListExists Then
+ ProgressPre.ChangeMax(.Count)
+ For Each jNode As EContainer In .Self
+ ProgressPre.Perform()
+ pid = New PostKV(String.Empty, jNode.Value({"node"}, "id"), PageBlock.Videos)
+ pid.Code = $"Stories:{pid.ID}"
+ nextCursor = jNode.Value("cursor")
+ If Not PostKvExists(pid) Then
+ newPostsDetected = True
+ PostsKVIDs.ListAddValue(pid, LNC)
+ _TempPostsList.Add(pid.Code)
+ _TempMediaList.ListAddValue(New UserMedia(String.Format(VideoHtmlUrlPattern, pid.ID),
+ UTypes.VideoPre) With {.Post = pid.ID}, LNC)
+ If Limit > 0 And _TempMediaList.Count >= Limit Then Exit Sub
+ Else
+ Exit Sub
+ End If
+ Next
+ End If
+ End With
+ End If
+ End Using
+ End If
+
+ If newPostsDetected And Not nextCursor.IsEmptyString Then DownloadData_Video(nextCursor, Token)
+ Catch ex As Exception
+ ProcessException(ex, Token, $"data (video) downloading error [{URL}]",, Responser)
+ End Try
+ End Sub
+ Private Sub DownloadData_Stories(ByVal Token As CancellationToken)
+ Dim URL$ = String.Empty
+ Const VarPattern$ = """UFI2CommentsProvider_commentsKey"":""StoriesSuspenseContentPaneRootWithEntryPointQuery"",""blur"":10,""bucketID"":""{0}"",""displayCommentsContextEnableComment"":true,""displayCommentsContextIsAdPreview"":false,""displayCommentsContextIsAggregatedShare"":false,""displayCommentsContextIsStorySet"":false,""displayCommentsFeedbackContext"":null,""feedbackSource"":65,""feedLocation"":""COMET_MEDIA_VIEWER"",""focusCommentID"":null,""initialBucketID"":""{0}"",""initialLoad"":true,""isInitialLoadFromCommentsNotification"":false,""isStoriesArchive"":false,""isStoryCommentingEnabled"":false,""scale"":1,""shouldDeferLoad"":false,""shouldEnableArmadilloStoryReply"":false,""shouldEnableLiveInStories"":true,""__relay_internal__pv__StoriesIsCommentEnabledrelayprovider"":false,""__relay_internal__pv__StoriesIsContextualReplyDisabledrelayprovider"":false,""__relay_internal__pv__StoriesIsShareToStoryEnabledrelayprovider"":false,""__relay_internal__pv__StoriesRingrelayprovider"":false,""__relay_internal__pv__StoriesLWRVariantrelayprovider"":""www_new_reactions"""
+ Try
+ Dim pUrl$, pUrlBase$
+ Dim pid As PostKV
+ Dim t As UTypes
+ Dim postDate As Date?
+
+ ValidateBaseTokens()
+ If StoryBucket.IsEmptyString Then Throw New ArgumentNullException("StoryBucket", "Unable to obtain StoryBucket")
+
+ URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Stories, Header_fb_fr_name_Stories,
+ SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg),
+ SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, StoryBucket) & "}"))
+
+ ResponserApplyDefs(Header_fb_fr_name_Stories)
+ ThrowAny(Token)
+
+ Dim r$ = Responser.GetResponse(URL)
+ If Not r.IsEmptyString Then r = RegexReplace(r, RParams.DM("[^\r\n]+", 0, EDP.ReturnValue))
+ If Not r.IsEmptyString Then
+ Using j As EContainer = JsonDocument.Parse(r)
+ If j.ListExists Then
+ With j({"data", "bucket", "unified_stories", "edges"})
+ If .ListExists Then
+ ProgressPre.ChangeMax(.Count)
+ For Each jNode As EContainer In .Self
+ ProgressPre.Perform()
+ With jNode({"node"})
+ If .ListExists Then
+ pid = New PostKV(.Value("id"), "", Sections.Stories)
+ With .ItemF({"attachments", 0, "media"})
+ If .ListExists Then
+ pid.ID = .Value("id")
+ pUrl = String.Empty
+ postDate = AConvert(Of Date)(.Value("creation_time"), UnixDate32Provider, Nothing)
+ Select Case .Value("__typename")
+ Case "Photo"
+ t = UTypes.Picture
+ pUrl = .Value({"image"}, "uri")
+ Case "Video"
+ t = UTypes.Video
+ pUrl = .Value("browser_native_hd_url").IfNullOrEmpty(.Value("browser_native_sd_url"))
+ End Select
+ If Not pUrl.IsEmptyString AndAlso Not PostKvExists(pid) Then
+ pUrlBase = $"https://www.facebook.com/stories/{StoryBucket}"
+ PostsKVIDs.Add(pid)
+ _TempMediaList.ListAddValue(New UserMedia(pUrl, t) With {
+ .URL_BASE = pUrlBase,
+ .File = CreateFileFromUrl(pUrl),
+ .SpecialFolder = $"{StoriesFolder} (user)",
+ .Post = New UserPost(pid.ID, postDate)}, LNC)
+ End If
+ End If
+ End With
+ End If
+ End With
+ Next
+ End If
+ End With
+ End If
+ End Using
+ End If
+ Catch ex As Exception
+ ProcessException(ex, Token, $"data (stories) downloading error [{URL}]",, Responser)
+ End Try
+ End Sub
+ Private Sub DownloadData_SavedPosts(ByVal Cursor As String, ByVal Token As CancellationToken)
+ Dim URL$ = String.Empty
+ Const VarPattern$ = """content_filter"":[],""count"":10,""cursor"":{0},""scale"":1,""use_case"":""SAVE_DEFAULT"""
+ Try
+ Dim nextCursor$ = String.Empty
+ Dim newPostsDetected As Boolean = False
+ Dim pUrl$, videoId$, imgUri$
+ Dim imgFile As SFile
+ Dim pid As PostKV
+
+ ValidateBaseTokens()
+ URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_SavedPosts, Header_fb_fr_name_SavedPosts,
+ SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg),
+ SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, If(Cursor.IsEmptyString, "null", $"""{Cursor}""")) & "}"))
+
+ ResponserApplyDefs(Header_fb_fr_name_SavedPosts)
+ ThrowAny(Token)
+
+ Dim r$ = Responser.GetResponse(URL)
+ If Not r.IsEmptyString Then
+ Using j As EContainer = JsonDocument.Parse(r)
+ If j.ListExists Then
+ With j({"data", "viewer", "saver_info", "all_saves", "edges"})
+ If .ListExists Then
+ ProgressPre.ChangeMax(.Count)
+ For Each jNode As EContainer In .Self
+ ProgressPre.Perform()
+ nextCursor = jNode.Value("cursor")
+ pid = New PostKV("", jNode.Value({"node"}, "id"), Sections.SavedPosts)
+ If Not PostKvExists(pid) Then
+ PostsKVIDs.Add(pid)
+ newPostsDetected = True
+ With jNode({"node", "savable"})
+ If .ListExists Then
+ pUrl = .Value("savable_permalink")
+ If Not pUrl.IsEmptyString Then
+ Select Case .Value("savable_default_category").StringToLower
+ Case "post_with_photo"
+ imgUri = .Value({"savable_image"}, "uri")
+ If Not imgUri.IsEmptyString Then
+ imgFile = CreateFileFromUrl(imgUri)
+ If Not imgFile.Name.IsEmptyString Then
+ ThrowAny(Token)
+ _TempMediaList.ListAddList(DownloadData_SavedPosts_ParseImagePost(pUrl, imgFile.Name, Token))
+ End If
+ End If
+ Case "video"
+ videoId = RegexReplace(pUrl, Regex_VideoIDFromURL)
+ If Not videoId.IsEmptyString Then _
+ _TempMediaList.ListAddValue(New UserMedia(pUrl, UTypes.VideoPre) With {.Post = videoId}, LNC)
+ Case Else : Continue For
+ End Select
+ End If
+ End If
+ End With
+ End If
+ Next
+ End If
+ End With
+ End If
+ End Using
+ End If
+
+ If newPostsDetected And Not nextCursor.IsEmptyString Then DownloadData_SavedPosts(nextCursor, Token)
+ Catch ex As Exception
+ ProcessException(ex, Token, $"data (saved posts) downloading error [{URL}]",, Responser)
+ End Try
+ End Sub
+ Private Function DownloadData_SavedPosts_ParseImagePost(ByVal PostUrl As String, ByVal ImageName As String, ByVal Token As CancellationToken,
+ Optional ByVal Round As Integer = 0) As IEnumerable(Of UserMedia)
+ Dim resp As Responser = HtmlResponserCreate()
+ Try
+ If Round > 0 Then ThrowAny(Token)
+ Dim script$, newUrl$
+ Dim jNode As EContainer, jNode2 As EContainer
+ Dim r$ = resp.GetResponse(PostUrl)
+
+ If Not r.IsEmptyString Then
+ script = RegexReplace(r, RParams.DMS($"