2024.4.14.0

Delete old notes and comments
API.Facebook: add app-id extraction from page; remove app-id from site requirements; update tokens parsing; update tokens regex
API.Instagram: add default function to parse tokens

Changelog.md

@@ -1,3 +1,10 @@
+# 2024.4.14.0
+
+*2024-04-14*
+
+- Fixed
+  - Facebook: can't get tokens
+
 # 2024.4.13.0
 
 *2024-04-13*

SCrawler.YouTube/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.13.0")>
+<Assembly: AssemblyVersion("2024.4.14.0")>
-<Assembly: AssemblyFileVersion("2024.4.13.0")>
+<Assembly: AssemblyFileVersion("2024.4.14.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb

@@ -709,7 +709,7 @@ Namespace API.YouTube.Objects
                     Bitrate = 0
                     _MediaType = UMTypes.Undefined
                     If SelectedVideoIndex >= 0 Then
-                        'URGENT: 2023.3.4 -> 2023.7.6
+                        '2023.3.4 -> 2023.7.6
                         'cmd.StringAppend($"bv*[format_id={SelectedVideo.ID}]")
                         cmd.StringAppend(SelectedVideo.ID)
                         _Size = SelectedVideo.Size
@@ -726,7 +726,7 @@ Namespace API.YouTube.Objects
                     End If
                     If SelectedAudioIndex >= 0 Then
                         Dim atCodec$
-                        'URGENT: 2023.3.4 -> 2023.7.6
+                        '2023.3.4 -> 2023.7.6
                         'cmd.StringAppend($"ba*[format_id={SelectedAudio.ID}]", "+")
                         cmd.StringAppend(SelectedAudio.ID, "+")
                         If OutputAudioCodec.StringToLower = ac3 Then
@@ -769,7 +769,7 @@ Namespace API.YouTube.Objects
                         subs = $"--write-subs --write-auto-subs --sub-format {OutputSubtitlesFormat.StringToLower} --sub-langs ""{subs}"" --convert-subs {OutputSubtitlesFormat.StringToLower}"
                     End If
                     If Not cmd.IsEmptyString Then
-                        'URGENT: 2023.3.4 -> 2023.7.6
+                        '2023.3.4 -> 2023.7.6
                         'cmd = $"yt-dlp -f ""{cmd}"""
                         'cmd = $"yt-dlp -f {cmd}"
                         cmd = $"{YTDLP_NAME} -f {cmd}"

SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.13.0")>
+<Assembly: AssemblyVersion("2024.4.14.0")>
-<Assembly: AssemblyFileVersion("2024.4.13.0")>
+<Assembly: AssemblyFileVersion("2024.4.14.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/API/Facebook/Declarations.vb

@@ -11,9 +11,8 @@ Imports PersonalUtilities.Functions.XML.Base
 Imports PersonalUtilities.Functions.RegularExpressions
 Namespace API.Facebook
     Friend Module Declarations
-        Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
-        Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
         Friend ReadOnly Regex_UserID As RParams = RParams.DMS("userid.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue)
+        Friend ReadOnly Regex_AppID As RParams = RParams.DMS("APP_ID.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue)
 
         Friend ReadOnly Regex_Photos_by As RParams = RParams.DMS("photos_by"",""id"":""([^""]+)", 1, EDP.ReturnValue)
         Friend ReadOnly Regex_FileName As RParams = RParams.DM("([^/\?]+\..{3,4})(?=(\?|\Z))", 0, EDP.ReturnValue)

SCrawler/API/Facebook/SiteSettings.vb

@@ -18,7 +18,7 @@ Namespace API.Facebook
 #Region "Auth"
         <PropertyOption(AllowNull:=False, ControlText:="Accept", ControlToolTip:="Header 'Accept'", IsAuth:=True), ControlNumber(21), PXML, PClonable>
         Friend ReadOnly Property Header_Accept As PropertyValue
-        <PropertyOption(ControlText:="x-ig-app-id", AllowNull:=True, IsAuth:=True)>
+        <PropertyOption(ControlText:="x-ig-app-id", AllowNull:=True, IsAuth:=True), HiddenControl>
         Friend Overrides ReadOnly Property HH_IG_APP_ID As PropertyValue
             Get
                 Return __HH_IG_APP_ID
@@ -74,7 +74,7 @@ Namespace API.Facebook
 #End Region
 #Region "BaseAuthExists, GetUserUrl, GetUserPostUrl, IsMyUser, IsMyImageVideo"
         Friend Overrides Function BaseAuthExists() As Boolean
-            Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value) And CBool(DownloadData_Impl.Value)
+            Return Responser.CookiesExists And CBool(DownloadData_Impl.Value) 'And ACheck(HH_IG_APP_ID.Value)
         End Function
         Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String
             Return DirectCast(User, UserData).GetProfileUrl

SCrawler/API/Facebook/UserData.vb

@@ -136,6 +136,7 @@ Namespace API.Facebook
         Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
             If CBool(MySettings.DownloadData_Impl.Value) Then
                 Try
+                    If Responser.Headers.Value(IG.Header_IG_APP_ID).IsEmptyString Then Responser.Headers.Remove(IG.Header_IG_APP_ID)
                     ResetBaseTokens()
                     GetUserTokens(Token)
                     LoadSavePostsKV(True)
@@ -529,8 +530,14 @@ Namespace API.Facebook
                 Dim r$ = resp.GetResponse(URL)
                 If Not r.IsEmptyString Then
                     If Responser.CookiesExists Then Responser.Cookies.Update(resp.Cookies)
-                    Token_dtsg = RegexReplace(r, Regex_UserToken_dtsg)
+                    ParseTokens(r, 0)
-                    Token_lsd = RegexReplace(r, Regex_UserToken_lsd)
+                    Dim app_id$ = RegexReplace(r, Regex_AppID)
+                    If Not app_id.IsEmptyString Then
+                        If Not AEquals(Of String)(MySettings.HH_IG_APP_ID.Value, app_id) Then
+                            MySettings.HH_IG_APP_ID.Value = app_id
+                            Responser.Headers.Add(IG.Header_IG_APP_ID, app_id)
+                        End If
+                    End If
                     Token_Photosby = RegexReplace(r, Regex_Photos_by)
                     If StoryBucket.IsEmptyString Then StoryBucket = RegexReplace(r, Regex_StoryBucket)
                     If ID.IsEmptyString Then
@@ -568,14 +575,14 @@ Namespace API.Facebook
                     .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "none"))
                     .Add("Sec-Fetch-User", "?1")
                     .Add("Upgrade-Insecure-Requests", 1)
-                    Dim h$ = Responser.Headers.Value(IG.Header_Browser)
+                    Dim cloneHeader As Action(Of String) = Sub(ByVal hName As String)
-                    If Not h.IsEmptyString Then .Add(IG.Header_Browser, h)
+                                                               Dim hValue$ = Responser.Headers.Value(hName)
-                    h = Responser.Headers.Value(IG.Header_BrowserExt)
+                                                               If Not hValue.IsEmptyString Then .Add(hName, hValue)
-                    If Not h.IsEmptyString Then .Add(IG.Header_BrowserExt, h)
+                                                           End Sub
-                    h = Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform))
+                    cloneHeader.Invoke(IG.Header_Browser)
-                    If Not h.IsEmptyString Then .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform, h))
+                    cloneHeader.Invoke(IG.Header_BrowserExt)
-                    h = Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion))
+                    cloneHeader.Invoke(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform).Name)
-                    If Not h.IsEmptyString Then .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion, h))
+                    cloneHeader.Invoke(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion).Name)
                     .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaMobile, "?0"))
                     .Add("Sec-Ch-Ua-Model", "")
                 End With

SCrawler/API/Instagram/Declarations.vb

@@ -18,6 +18,8 @@ Namespace API.Instagram
         Friend ReadOnly ObtainMedia_SizeFuncPic_RegexP As RParams = RParams.DMS("_p(\d+)x(\d+)", 1, EDP.ReturnValue)
         Friend ReadOnly ObtainMedia_SizeFuncPic_RegexS As RParams = RParams.DMS("_s(\d+)x(\d+)", 1, EDP.ReturnValue)
         Friend Const PageTokenRegexPatternDefault As String = "\[\],{""token"":""(.*?)""},\d+\]"
+        Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
+        Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
         Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser, ByVal UpdateWwwClaim As Boolean)
             Const r_wwwClaimName$ = "x-ig-set-www-claim"
             Const r_tokenName$ = SiteSettings.Header_CSRF_TOKEN_COOKIE

SCrawler/API/Instagram/UserData.GQL.vb

@@ -297,7 +297,18 @@ Namespace API.Instagram
                     End With
                 End With
                 Dim r$ = Responser.GetResponse(MySiteSettings.GetUserUrl(Me))
+                ParseTokens(r, 0)
+            Catch ex As Exception
+            Finally
+                ChangeResponserMode(_UseGQL, Not _UseGQL)
+            End Try
+        End Sub
+        Protected Sub ParseTokens(ByVal r As String, ByVal Attempt As Integer)
+            Try
                 If Not r.IsEmptyString Then
+                    ResetBaseTokens()
+                    Select Case Attempt
+                        Case 0
                             Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)
                             Dim tokens As List(Of String) = RegexReplace(r, rr)
                             Dim tt$, ttVal$
@@ -322,10 +333,13 @@ Namespace API.Instagram
                                     End If
                                 Next
                             End If
+                        Case 1
+                            Token_dtsg = RegexReplace(r, Regex_UserToken_dtsg)
+                            Token_lsd = RegexReplace(r, Regex_UserToken_lsd)
+                    End Select
+                    If Not ValidateBaseTokens() And Attempt = 0 Then ParseTokens(r, Attempt + 1)
                 End If
-            Catch ex As Exception
+            Catch
-            Finally
-                ChangeResponserMode(_UseGQL, Not _UseGQL)
             End Try
         End Sub
 #End Region

SCrawler/API/PornHub/UserData.vb

@@ -29,7 +29,6 @@ Namespace API.PornHub
         Private Const Name_DownloadFavorite As String = "DownloadFavorite"
         Private Const Name_DownloadGifs As String = "DownloadGifs"
         Private Const Name_DownloadPhotoOnlyFromModelHub As String = "DownloadPhotoOnlyFromModelHub"
-        <Obsolete> Private Const Name_IsUser As String = "IsUser"
 #End Region
 #Region "Structures"
         Private Structure FlashVar : Implements IRegExCreator
@@ -254,14 +253,7 @@ Namespace API.PornHub
                     DownloadFavorite = .Value(Name_DownloadFavorite).FromXML(Of Boolean)(False)
                     DownloadGifs = .Value(Name_DownloadGifs).FromXML(Of Integer)(False)
                     DownloadPhotoOnlyFromModelHub = .Value(Name_DownloadPhotoOnlyFromModelHub).FromXML(Of Boolean)(True)
-                    If .Contains(Name_SiteMode) Then
                     SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User)
-                    Else
-                        'TODELETE: PornHub 'IsUser' 20231113
-#Disable Warning BC40008
-                        SiteMode = IIf(.Value(Name_IsUser).FromXML(Of Boolean)(True), SiteModes.User, SiteModes.Search)
-#Enable Warning
-                    End If
                     UpdateUserOptions()
                 Else
                     If UpdateUserOptions() Then .Value(Name_LabelsName) = LabelsString
@@ -404,7 +396,6 @@ Namespace API.PornHub
                 Dim r$ = Responser.GetResponse(URL)
                 If Not r.IsEmptyString Then
                     Dim l As List(Of UserVideo) = RegexFields(Of UserVideo)(r, {RegexUserVideos}, {6, 7, 3, 10})
-                    'URGENT: PornHub: changed list trimming
                     'If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(3, l.Count).ToList
                     If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(1, l.Count).ToList
                     If l.ListExists Then

SCrawler/API/ThreadsNet/SiteSettings.vb

@@ -103,8 +103,6 @@ Namespace API.ThreadsNet
 
             With Responser
                 .Accept = "*/*"
-                'URGENT: remove after debug
-                .DeclaredError = EDP.SendToLog + EDP.ThrowException
                 If .UserAgentExists Then useragent = .UserAgent
                 With .Headers
                     If .Count > 0 Then
@@ -125,6 +123,8 @@ Namespace API.ThreadsNet
                     .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "cors"))
                     .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "same-origin"))
                     .Add("Sec-Fetch-User", "?1")
+                    .Add("dht", 1)
+                    .Add("drp", 1)
                     .Add(Instagram.UserData.GQL_HEADER_FB_FRINDLY_NAME, "BarcelonaProfileThreadsTabRefetchableQuery")
                 End With
                 .CookiesExtractMode = Responser.CookiesExtractModes.Any

SCrawler/API/ThreadsNet/UserData.vb

@@ -157,38 +157,22 @@ Namespace API.ThreadsNet
         Private Function UpdateCredentials(Optional ByVal e As ErrorsDescriber = Nothing) As Boolean
             Dim URL$ = $"https://www.threads.net/@{NameTrue}"
             ResetBaseTokens()
+            Dim headers As New HttpHeaderCollection
+            headers.AddRange(Responser.Headers)
             Try
-                Responser.Method = "GET"
+                With Responser
-                Responser.Referer = URL
+                    .Method = "GET"
-                Responser.Headers.Remove(GQL_HEADER_FB_LSD)
+                    .Referer = URL
+                    With .Headers
+                        .Remove(GQL_HEADER_FB_LSD)
+                        .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, "document"))
+                        .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "navigate"))
+                    End With
+                End With
                 WaitTimer()
                 Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException)
-                Dim rr As RParams
-                Dim tt$, ttVal$
                 If Not r.IsEmptyString Then
-                    rr = RParams.DM(Instagram.PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)
+                    ParseTokens(r, 0)
-                    Dim tokens As List(Of String) = RegexReplace(r, rr)
-                    If tokens.ListExists Then
-                        With rr
-                            .Match = Nothing
-                            .MatchSub = 1
-                            .WhatGet = RegexReturn.Value
-                        End With
-                        For Each tt In tokens
-                            If Not Token_dtsg.IsEmptyString And Not Token_lsd.IsEmptyString Then
-                                Exit For
-                            Else
-                                ttVal = RegexReplace(tt, rr)
-                                If Not ttVal.IsEmptyString Then
-                                    If ttVal.Contains(":") Then
-                                        If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal
-                                    Else
-                                        If Token_lsd.IsEmptyString Then Token_lsd = ttVal
-                                    End If
-                                End If
-                            End If
-                        Next
-                    End If
                     If ID.IsEmptyString Then ID = RegexReplace(r, RParams.DMS("""props"":\{""user_id"":""(\d+)""\},", 1, EDP.ReturnValue))
                 End If
                 Return Valid
@@ -204,6 +188,12 @@ Namespace API.ThreadsNet
                 'LogError(ex, $"failed to update some{IIf(notFound.IsEmptyString, String.Empty, $" ({notFound})")} credentials", e)
                 LogError(eex, String.Empty, e)
                 Return False
+            Finally
+                If headers.ListExists Then
+                    Responser.Headers.Clear()
+                    Responser.Headers.AddRange(headers)
+                    headers.Dispose()
+                End If
             End Try
         End Function
 #End Region

SCrawler/API/Xhamster/UserData.vb

@@ -280,32 +280,18 @@ Namespace API.Xhamster
                     End If
                 End If
 
-                'TODELETE: xHamster remove old container nodes attachments 
-
                 If IsSavedPosts Then
                     URL = $"https://xhamster.com/my/favorites/{IIf(IsVideo, "videos", "photos-and-galleries")}{IIf(Page = 1, String.Empty, $"/{Page}")}"
                     containerNodes.Add(If(IsVideo, {"favoriteVideoListComponent", "models"}, {"favoritesGalleriesAndPhotosCollection"}))
                 ElseIf IsChannel Then
                     URL = $"https://xhamster.com/channels/{TrueName}/newest{IIf(Page = 1, String.Empty, $"/{Page}")}"
-                    'containerNodes.Add({"trendingVideoListComponent", "models"})
-                    'containerNodes.Add({"pagesCategoryComponent", "trendingVideoListProps", "models"})
                 ElseIf SiteMode = SiteModes.Search Then
                     URL = GetNonUserUrl(Page)
                     containerNodes.Add({"searchResult", "models"})
                 ElseIf IsCreator Or SiteMode = SiteModes.Tags Or SiteMode = SiteModes.Categories Or SiteMode = SiteModes.Pornstars Then
                     URL = GetNonUserUrl(Page)
-                    'If SiteMode = SiteModes.Pornstars Then
-                    '    containerNodes.Add({"trendingVideoListComponent", "models"})
-                    '    containerNodes.Add({"pagesCategoryComponent", "trendingVideoListProps", "models"})
-                    'Else
-                    '    containerNodes.Add({"pagesCategoryComponent", "trendingVideoListProps", "models"})
-                    '    containerNodes.Add({"trendingVideoListComponent", "models"})
-                    'End If
-                    'containerNodes.Add({"trendingVideoSectionComponent", "videoModels"})
                 Else
                     URL = $"https://xhamster.com/users/{TrueName}/{IIf(IsVideo, "videos", "photos")}{IIf(Page = 1, String.Empty, $"/{Page}")}"
-                    'containerNodes.Add({If(IsVideo, "userVideoCollection", "userGalleriesCollection")})
-                    'containerNodes.Add(If(IsVideo, {"videoListComponent", "models"}, {"userGalleriesCollection"}))
                 End If
                 ThrowAny(Token)
 

SCrawler/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.13.0")>
+<Assembly: AssemblyVersion("2024.4.14.0")>
-<Assembly: AssemblyFileVersion("2024.4.13.0")>
+<Assembly: AssemblyFileVersion("2024.4.14.0")>
 <Assembly: NeutralResourcesLanguage("en")>

Tools/NET.FrameworkVersion.ps1

@@ -1,4 +1,5 @@
-$arr=Get-ChildItem -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework\|Select-Object Name
+try{Set-ExecutionPolicy Unrestricted CurrentUser -ErrorAction SilentlyContinue}catch{}
+$arr=Get-ChildItem -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework\|Select-Object Name
 $mv=0
 $found=0
 foreach($v in $arr){if($v.Name -match "v4.0."){$mv=$v.Name}}