2023.10.1.0

JustForFans: some profiles won't download

Changelog.md

@@ -1,3 +1,25 @@
+# 2023.10.1.0
+
+*2023-10-01*
+
+- Added
+  - **Threads.net**
+  - YouTube: add URL standardization
+- Fixed
+  - UserEditor: disable updating labels if they haven't changed
+  - Collections: incorrect updating of colors and labels when adding a new user
+  - RedGifs: incorrect handling of error 410
+  - Mastodon: hide error 503
+  - JustForFans: some profiles won't download
+  - Minor bugs
+
+# 2023.9.21.0
+
+*2023-09-21*
+
+- Fixed
+  - PornHub: videos are not downloading
+
 # 2023.9.20.0
 
 *2023-09-20*

README.md

@@ -11,7 +11,7 @@
 :eu:
 :greece:
 
-A program to download photo and video from [any site](#supported-sites) (e.g. YouTube, YouTube Music, OnlyFans, Reddit, Twitter, Mastodon, Instagram, TikTok, RedGifs, JustForFans, PornHub, XHamster, XVIDEOS, ThisVid, LPSG, Pinterest).
+A program to download photo and video from [any site](#supported-sites) (e.g. YouTube, YouTube Music, OnlyFans, Reddit, Twitter, Mastodon, Instagram, Threads, TikTok, RedGifs, JustForFans, PornHub, XHamster, XVIDEOS, ThisVid, LPSG, Pinterest).
 
 **If you like SCrawler, please like the program on [this site](https://alternativeto.net/software/scrawler/about/) and/or [this](https://www.softpedia.com/get/Internet/Download-Managers/Social-networks-crawler.shtml)**
 <!---Do you like this program? Consider adding to my coffee fund by making a donation to show your support. :blush:
@@ -37,6 +37,7 @@ A program to download photo and video from [any site](#supported-sites) (e.g. Yo
   - JustForFans images and videos, saved (bookmarked) posts;
   - Mastodon images and videos, saved (bookmarked) posts;
   - Instagram images and videos, tagged posts, stories, saved posts;
+  - Threads images and videos;
   - TikTok videos;
   - Pinterest boards, users, saved posts;
   - Imgur images, galleries and videos;
@@ -73,6 +74,7 @@ A program to download photo and video from [any site](#supported-sites) (e.g. Yo
 - **OnlyFans**
 - **Mastodon**
 - **Instagram**
+- **Threads**
 - JustForFans
 - TikTok
 - RedGifs
@@ -124,6 +126,7 @@ First, the program downloads the full profile. After the program downloads only
   - [OnlyFans](https://github.com/AAndyProgram/SCrawler/wiki/Settings#onlyfans)
   - [Mastodon](https://github.com/AAndyProgram/SCrawler/wiki/Settings#mastodon)
   - [Instagram](https://github.com/AAndyProgram/SCrawler/wiki/Settings#instagram)
+  - [Threads](https://github.com/AAndyProgram/SCrawler/wiki/Settings#threads)
   - [JustForFans](https://github.com/AAndyProgram/SCrawler/wiki/Settings#justforfans)
   - [TikTok](https://github.com/AAndyProgram/SCrawler/wiki/Settings#tiktok)
   - [RedGifs](https://github.com/AAndyProgram/SCrawler/wiki/Settings#redgifs)

SCrawler.PluginProvider/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2023.8.6.0")>
-<Assembly: AssemblyFileVersion("2023.8.6.0")>
+<Assembly: AssemblyVersion("2023.10.1.0")>
+<Assembly: AssemblyFileVersion("2023.10.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.PluginProvider/ObjectInterfaces/ILogProvider.vb

@@ -11,6 +11,6 @@ Namespace Plugin
         Overloads Sub Add(ByVal Message As String)
         Overloads Sub Add(ByVal ex As Exception, ByVal Message As String,
                           Optional ByVal ShowMainMsg As Boolean = False, Optional ByVal ShowErrorMsg As Boolean = False,
-                          Optional ByVal SendInLog As Boolean = True)
+                          Optional ByVal SendToLog As Boolean = True)
     End Interface
 End Namespace

SCrawler.YouTube/Base/YouTubeFunctions.vb

@@ -20,6 +20,29 @@ Namespace API.YouTube.Base
         Public Const UrlTypePattern As String = "(?<=https?://[^/]*?youtube.com/)((@|[^\?/&]+))([/\?]{0,1}(list=|v=|)([^\?/&]*))(?=(\S+|\Z|))"
         Private Sub New()
         End Sub
+        Public Shared Function StandardizeURL(ByVal URL As String) As String
+            Try
+                Dim isMusic As Boolean = False, isShorts As Boolean = False
+                If Info_GetUrlType(URL, isMusic, isShorts) = YouTubeMediaType.Single Then
+                    If Not isMusic And Not isShorts Then
+                        Dim videoOptionRegex As RParams = RParams.DMS("[\?&]v=([^\?&]+)", 1, EDP.ReturnValue)
+                        Dim data As List(Of String) = RegexReplace(URL, RParams.DMS(UrlTypePattern, 0, RegexReturn.ListByMatch, EDP.ReturnValue))
+                        Dim val$ = String.Empty
+                        If data.ListExists Then
+                            For Each d$ In data
+                                val = RegexReplace(d, videoOptionRegex)
+                                If Not val.IsEmptyString Then Exit For
+                            Next
+                            data.Clear()
+                        End If
+                        If Not val.IsEmptyString Then Return $"https://www.youtube.com/watch?v={val}"
+                    End If
+                End If
+                Return URL
+            Catch ex As Exception
+                Return URL
+            End Try
+        End Function
         Public Shared Function IsMyUrl(ByVal URL As String) As Boolean
             Return Not Info_GetUrlType(URL) = YouTubeMediaType.Undefined
         End Function

SCrawler.YouTube/Base/YouTubeSettings.vb

@@ -133,6 +133,9 @@ Namespace API.YouTube.Base
         End Property
 #End Region
 #Region "Defaults"
+        <Browsable(True), GridVisible, XMLVN({"Defaults"}, True), Category("Defaults"), DisplayName("Standardize URLs"),
+            Description("Standardize URLs by eliminating unwanted strings. Default: true.")>
+        Public ReadOnly Property StandardizeURLs As XMLValue(Of Boolean)
         <Browsable(True), GridVisible, XMLVN({"Defaults"}), Category("Defaults"), DisplayName("Replace modification date"),
             Description("Set the file date to the date the video was added (website) (if available). Default: false.")>
         Public ReadOnly Property ReplaceModificationDate As XMLValue(Of Boolean)

SCrawler.YouTube/Controls/VideoOptionsForm.vb

@@ -433,7 +433,7 @@ Namespace API.YouTube.Controls
         End Sub
 #End Region
 #Region "Footer"
-        Private Sub BTT_BROWSE_MouseClick(sender As Object, e As MouseEventArgs) Handles BTT_BROWSE.MouseClick
+        Private Sub BTT_BROWSE_MouseDown(sender As Object, e As MouseEventArgs) Handles BTT_BROWSE.MouseDown
             Dim f As SFile
 #Disable Warning BC40000
             If MyContainer.HasElements Then

SCrawler.YouTube/Downloader/VideoListForm.vb

@@ -247,6 +247,8 @@ Namespace DownloadObjects.STDownloader
                 If e.Control Then useCookies = True
                 Dim useCookiesParse As Boolean? = Nothing
                 If useCookies Then useCookiesParse = True
+                Dim standardizeUrls As Boolean = MyYouTubeSettings.StandardizeURLs
+                Dim standardize As Func(Of String, String) = Function(input) If(standardizeUrls, YouTubeFunctions.StandardizeURL(input), input)
 
                 Dim c As IYouTubeMediaContainer = Nothing
                 Dim url$ = String.Empty
@@ -264,7 +266,7 @@ Namespace DownloadObjects.STDownloader
                                     pForm.SetInitialValues(.Count, "Parsing playlists...")
                                     Dim containers As New List(Of IYouTubeMediaContainer)
                                     For Each u$ In .Self
-                                        containers.Add(YouTubeFunctions.Parse(u, useCookiesParse, pForm.Token, pForm.MyProgress, True, False))
+                                        containers.Add(YouTubeFunctions.Parse(standardize(u), useCookiesParse, pForm.Token, pForm.MyProgress, True, False))
                                         pForm.NextPlaylist()
                                         pForm.MyProgress.Perform()
                                     Next
@@ -295,7 +297,7 @@ Namespace DownloadObjects.STDownloader
                         pForm = New ParsingProgressForm
                         pForm.Show(Me)
                         pForm.SetInitialValues(1, "Parsing data...")
-                        c = YouTubeFunctions.Parse(url, useCookiesParse, pForm.Token, pForm.MyProgress, GetDefault, GetShorts)
+                        c = YouTubeFunctions.Parse(standardize(url), useCookiesParse, pForm.Token, pForm.MyProgress, GetDefault, GetShorts)
                         pForm.Dispose()
                     End If
                     If Not c Is Nothing Then

SCrawler.YouTube/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2023.9.20.0")>
-<Assembly: AssemblyFileVersion("2023.9.20.0")>
+<Assembly: AssemblyVersion("2023.10.1.0")>
+<Assembly: AssemblyFileVersion("2023.10.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2023.9.20.0")>
-<Assembly: AssemblyFileVersion("2023.9.20.0")>
+<Assembly: AssemblyVersion("2023.10.1.0")>
+<Assembly: AssemblyFileVersion("2023.10.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/API/Base/UserDataBase.vb

@@ -1226,7 +1226,7 @@ BlockNullPicture:
                 End If
                 ThrowIfDisposed()
                 If Not _PictureExists Or _EnvirInvokeUserUpdated Then OnUserUpdated()
-            Catch oex As OperationCanceledException When Token.IsCancellationRequested Or TokenPersonal.IsCancellationRequested
+            Catch oex As OperationCanceledException When Token.IsCancellationRequested Or TokenPersonal.IsCancellationRequested Or TokenQueue.IsCancellationRequested
                 MyMainLOG = $"{ToStringForLog()}: downloading canceled"
                 Canceled = True
             Catch exit_ex As ExitException
@@ -1829,6 +1829,7 @@ BlockNullPicture:
                         If m.Contains(IUserData.EraseMode.History) Then
                             If MyFilePosts.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True
                             If MyFileData.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True
+                            EraseData_AdditionalDataFiles()
                         End If
                         If m.Contains(IUserData.EraseMode.Data) Then
                             Dim files As List(Of SFile) = SFile.GetFiles(DownloadContentDefault_GetRootDir.CSFileP,, SearchOption.AllDirectories, e)
@@ -1850,6 +1851,8 @@ BlockNullPicture:
                 Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, $"EraseData({CInt(Mode)}): {ToStringForLog()}", False)
             End Try
         End Function
+        Protected Overridable Sub EraseData_AdditionalDataFiles()
+        End Sub
         Friend Overridable Function Delete(Optional ByVal Multiple As Boolean = False, Optional ByVal CollectionValue As Integer = -1) As Integer Implements IUserData.Delete
             Dim f As SFile = SFile.GetPath(MyFile.CutPath.Path)
             If f.Exists(SFO.Path, False) AndAlso (User.Merged OrElse f.Delete(SFO.Path, Settings.DeleteMode)) Then
@@ -2026,8 +2029,8 @@ BlockNullPicture:
         End Function
 #End Region
 #Region "Errors functions"
-        Protected Sub LogError(ByVal ex As Exception, ByVal Message As String)
-            ErrorsDescriber.Execute(EDP.SendToLog, ex, $"{ToStringForLog()}: {Message}")
+        Protected Sub LogError(ByVal ex As Exception, ByVal Message As String, Optional ByVal e As ErrorsDescriber = Nothing)
+            ErrorsDescriber.Execute(If(e.Exists, e, New ErrorsDescriber(EDP.SendToLog)), ex, $"{ToStringForLog()}: {Message}")
         End Sub
         Protected Sub ErrorDownloading(ByVal f As SFile, ByVal URL As String)
             If Not f.Exists Then MyMainLOG = $"Error downloading from [{URL}] to [{f}]"
@@ -2040,9 +2043,13 @@ BlockNullPicture:
         Private Overloads Sub ThrowAny() Implements IThrower.ThrowAny
             ThrowAny(TokenQueue)
         End Sub
+        ''' <summary><c>ThrowAnyImpl(Token)</c></summary>
         ''' <exception cref="OperationCanceledException"></exception>
         ''' <exception cref="ObjectDisposedException"></exception>
         Friend Overridable Overloads Sub ThrowAny(ByVal Token As CancellationToken)
+            ThrowAnyImpl(Token)
+        End Sub
+        Protected Sub ThrowAnyImpl(ByVal Token As CancellationToken)
             Token.ThrowIfCancellationRequested()
             TokenQueue.ThrowIfCancellationRequested()
             TokenPersonal.ThrowIfCancellationRequested()

SCrawler/API/Instagram/Declarations.vb

@@ -17,7 +17,7 @@ Namespace API.Instagram
         Friend ReadOnly FilesPattern As RParams = RParams.DMS(".+?([^/\?]+?\.[\w\d]{3,4})(?=(\?|\Z))", 1, EDP.ReturnValue)
         Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser)
             Const r_wwwClaimName$ = "x-ig-set-www-claim"
-            Const r_tokenName$ = "csrftoken"
+            Const r_tokenName$ = SiteSettings.Header_CSRF_TOKEN_COOKIE
             If Not Source Is Nothing Then
                 Dim isInternal As Boolean = TypeOf Source Is WebDataResponse
                 Dim wwwClaimName$, tokenName$

SCrawler/API/Instagram/SiteSettings.vb

@@ -70,13 +70,14 @@ Namespace API.Instagram
         End Class
 #End Region
 #Region "Authorization properties"
-        Private Const Header_IG_APP_ID As String = "x-ig-app-id"
+        Friend Const Header_IG_APP_ID As String = "x-ig-app-id"
         Friend Const Header_IG_WWW_CLAIM As String = "x-ig-www-claim"
         Friend Const Header_CSRF_TOKEN As String = "x-csrftoken"
-        Private Const Header_ASBD_ID As String = "X-Asbd-Id"
-        Private Const Header_Browser As String = "Sec-Ch-Ua"
-        Private Const Header_BrowserExt As String = "Sec-Ch-Ua-Full-Version-List"
-        Private Const Header_Platform As String = "Sec-Ch-Ua-Platform-Version"
+        Friend Const Header_CSRF_TOKEN_COOKIE As String = "csrftoken"
+        Friend Const Header_ASBD_ID As String = "X-Asbd-Id"
+        Friend Const Header_Browser As String = "Sec-Ch-Ua"
+        Friend Const Header_BrowserExt As String = "Sec-Ch-Ua-Full-Version-List"
+        Friend Const Header_Platform As String = "Sec-Ch-Ua-Platform-Version"
         <PropertyOption(ControlText:="Hash", ControlToolTip:="Instagram session hash for tagged posts", IsAuth:=True), PXML("InstaHash"), ControlNumber(0)>
         Friend ReadOnly Property HashTagged As PropertyValue
         <PropertyOption(ControlText:="x-csrftoken", IsAuth:=True, AllowNull:=False), ControlNumber(2)>
@@ -365,13 +366,16 @@ Namespace API.Instagram
             SkipUntilNextSession = False
         End Sub
 #End Region
-#Region "UserOptions, GetUserPostUrl"
+#Region "UserOptions, GetUserUrl, GetUserPostUrl"
         Friend Overrides Sub UserOptions(ByRef Options As Object, ByVal OpenForm As Boolean)
             If Options Is Nothing OrElse Not TypeOf Options Is EditorExchangeOptions Then Options = New EditorExchangeOptions(Me)
             If OpenForm Then
                 Using f As New InternalSettingsForm(Options, Me, False) : f.ShowDialog() : End Using
             End If
         End Sub
+        Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String
+            Return String.Format(UrlPatternUser, DirectCast(User, UserData).NameTrue)
+        End Function
         Friend Overrides Function GetUserPostUrl(ByVal User As UserDataBase, ByVal Media As UserMedia) As String
             Try
                 Dim code$ = DirectCast(User, UserData).GetPostCodeById(Media.Post.ID)

SCrawler/API/Instagram/UserData.vb

@@ -30,7 +30,7 @@ Namespace API.Instagram
         Private Const Name_NameTrue As String = "NameTrue"
 #End Region
 #Region "Declarations"
-        Private Structure PostKV : Implements IEContainerProvider
+        Protected Structure PostKV : Implements IEContainerProvider
             Private Const Name_Code As String = "Code"
             Private Const Name_Section As String = "Section"
             Friend Code As String
@@ -78,8 +78,8 @@ Namespace API.Instagram
         Friend Property GetStories As Boolean
         Friend Property GetStoriesUser As Boolean
         Friend Property GetTaggedData As Boolean
-        Private _NameTrue As String = String.Empty
-        Private ReadOnly Property NameTrue As String
+        Protected _NameTrue As String = String.Empty
+        Friend ReadOnly Property NameTrue As String
             Get
                 Return _NameTrue.IfNullOrEmpty(Name)
             End Get
@@ -143,12 +143,22 @@ Namespace API.Instagram
                 Throw New ExitException
             End Sub
         End Class
-        Private Sub LoadSavePostsKV(ByVal Load As Boolean)
+        Private ReadOnly Property MyFilePostsKV As SFile
+            Get
+                Dim f As SFile = MyFilePosts
+                If Not f.IsEmptyString Then
+                    f.Name &= "_KV"
+                    f.Extension = "xml"
+                    Return f
+                Else
+                    Return Nothing
+                End If
+            End Get
+        End Property
+        Protected Sub LoadSavePostsKV(ByVal Load As Boolean)
             Dim x As XmlFile
-            Dim f As SFile = MyFilePosts
+            Dim f As SFile = MyFilePostsKV
             If Not f.IsEmptyString Then
-                f.Name &= "_KV"
-                f.Extension = "xml"
                 If Load Then
                     PostsKVIDs.Clear()
                     x = New XmlFile(f, Protector.Modes.All, False) With {.AllowSameNames = True, .XmlReadOnly = True}
@@ -182,10 +192,8 @@ Namespace API.Instagram
         Friend Function GetPostCodeById(ByVal PostID As String) As String
             Try
                 If Not PostID.IsEmptyString Then
-                    Dim f As SFile = MyFilePosts
+                    Dim f As SFile = MyFilePostsKV
                     If Not f.IsEmptyString Then
-                        f.Name &= "_KV"
-                        f.Extension = "xml"
                         Dim l As List(Of PostKV) = Nothing
                         Using x As New XmlFile(f, Protector.Modes.All, False) With {.AllowSameNames = True, .XmlReadOnly = True}
                             x.LoadData()
@@ -213,11 +221,15 @@ Namespace API.Instagram
             End If
         End Function
         Private _DownloadingInProgress As Boolean = False
+        Private _Limit As Integer = -1
+        Private _TotalPostsParsed As Integer = 0
         Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
             UserNameRequested = False
             Dim s As Sections = Sections.Timeline
             Dim errorFound As Boolean = False
             Try
+                _Limit = If(DownloadTopCount, -1)
+                _TotalPostsParsed = 0
                 LoadSavePostsKV(True)
                 _DownloadingInProgress = True
                 AddHandler Responser.ResponseReceived, AddressOf Responser_ResponseReceived
@@ -270,7 +282,7 @@ Namespace API.Instagram
             Catch ex As Exception
             End Try
         End Sub
-        Private Sub UpdateResponser()
+        Protected Overridable Sub UpdateResponser()
             Try
                 If _DownloadingInProgress AndAlso Not Responser Is Nothing AndAlso Not Responser.Disposed Then
                     _DownloadingInProgress = False
@@ -280,10 +292,10 @@ Namespace API.Instagram
             Catch
             End Try
         End Sub
-        Private Sub Responser_ResponseReceived(ByVal Sender As Object, ByVal e As EventArguments.WebDataResponse)
+        Protected Overridable Sub Responser_ResponseReceived(ByVal Sender As Object, ByVal e As EventArguments.WebDataResponse)
             Declarations.UpdateResponser(e, Responser)
         End Sub
-        Private Enum Sections : Timeline : Tagged : Stories : UserStories : SavedPosts : End Enum
+        Protected Enum Sections : Timeline : Tagged : Stories : UserStories : SavedPosts : End Enum
         Private Const StoriesFolder As String = "Stories"
         Private Const TaggedFolder As String = "Tagged"
 #Region "429 bypass"
@@ -572,6 +584,7 @@ Namespace API.Instagram
             Dim URL$ = String.Empty
             Dim dValue% = 1
             Dim _Index% = 0
+            Dim before%
             If PostsToReparse.Count > 0 Then ProgressPre.ChangeMax(PostsToReparse.Count)
             Try
                 Do While dValue = 1
@@ -600,7 +613,12 @@ Namespace API.Instagram
                                     If Not j Is Nothing Then
                                         If If(j("items")?.Count, 0) > 0 Then
                                             With j("items")
-                                                For Each jj In .Self : ObtainMedia(jj, PostsToReparse(i).ID) : Next
+                                                For Each jj In .Self
+                                                    before = _TempMediaList.Count
+                                                    ObtainMedia(jj, PostsToReparse(i).ID)
+                                                    If Not before = _TempMediaList.Count Then _TotalPostsParsed += 1
+                                                    If _Limit > 0 And _TotalPostsParsed >= _Limit Then Throw New ExitException
+                                                Next
                                             End With
                                         End If
                                         j.Dispose()
@@ -643,13 +661,17 @@ Namespace API.Instagram
                 End Using
             End If
         End Sub
-        Private Function DefaultParser(ByVal Items As IEnumerable(Of EContainer), ByVal Section As Sections, ByVal Token As CancellationToken,
-                                       Optional ByVal SpecFolder As String = Nothing) As Boolean
+        Protected DefaultParser_ElemNode() As Object = Nothing
+        Protected DefaultParser_IgnorePass As Boolean = False
+        Protected DefaultParser_PostUrlCreator As Func(Of PostKV, String) = Function(post) $"https://www.instagram.com/p/{post.Code}/"
+        Protected Function DefaultParser(ByVal Items As IEnumerable(Of EContainer), ByVal Section As Sections, ByVal Token As CancellationToken,
+                                         Optional ByVal SpecFolder As String = Nothing) As Boolean
             ThrowAny(Token)
             If Items.Count > 0 Then
                 Dim PostIDKV As PostKV
                 Dim Pinned As Boolean
-                Dim PostDate$
+                Dim PostDate$, PostOriginUrl$
+                Dim before%
                 If SpecFolder.IsEmptyString Then
                     Select Case Section
                         Case Sections.Tagged : SpecFolder = TaggedFolder
@@ -660,22 +682,26 @@ Namespace API.Instagram
                 ProgressPre.ChangeMax(Items.Count)
                 For Each nn In Items
                     ProgressPre.Perform()
-                    With nn
+                    With If(Not DefaultParser_ElemNode Is Nothing, nn.ItemF(DefaultParser_ElemNode), nn)
                         PostIDKV = New PostKV(.Value("code"), .Value("id"), Section)
+                        PostOriginUrl = DefaultParser_PostUrlCreator(PostIDKV)
                         Pinned = .Contains("timeline_pinned_user_ids")
-                        If PostKvExists(PostIDKV) Then
+                        If Not DefaultParser_IgnorePass AndAlso PostKvExists(PostIDKV) Then
                             If Not Pinned Then Return False
                         Else
                             _TempPostsList.Add(PostIDKV.ID)
                             PostsKVIDs.ListAddValue(PostIDKV, LNC)
                             PostDate = .Value("taken_at")
-                            If Not IsSavedPosts Then
+                            If Not DefaultParser_IgnorePass And Not IsSavedPosts Then
                                 Select Case CheckDatesLimit(PostDate, UnixDate32Provider)
                                     Case DateResult.Skip : Continue For
                                     Case DateResult.Exit : If Not Pinned Then Return False
                                 End Select
                             End If
-                            ObtainMedia(.Self, PostIDKV.ID, SpecFolder, PostDate)
+                            before = _TempMediaList.Count
+                            ObtainMedia(.Self, PostIDKV.ID, SpecFolder, PostDate,, PostOriginUrl)
+                            If Not before = _TempMediaList.Count Then _TotalPostsParsed += 1
+                            If _Limit > 0 And _TotalPostsParsed >= _Limit Then Return False
                         End If
                     End With
                 Next
@@ -686,7 +712,7 @@ Namespace API.Instagram
         End Function
 #End Region
 #Region "Code ID converters"
-        Private Function CodeToID(ByVal Code As String) As String
+        Protected Function CodeToID(ByVal Code As String) As String
             Const CodeSymbols$ = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
             Try
                 If Not Code.IsEmptyString Then
@@ -706,12 +732,19 @@ Namespace API.Instagram
         End Function
 #End Region
 #Region "Obtain Media"
-        Private Sub ObtainMedia(ByVal n As EContainer, ByVal PostID As String, Optional ByVal SpecialFolder As String = Nothing,
-                                Optional ByVal DateObj As String = Nothing)
+        Protected ObtainMedia_SizeFuncVid As Func(Of EContainer, Sizes) = Nothing
+        Protected ObtainMedia_SizeFuncPic As Func(Of EContainer, Sizes) = Nothing
+        Protected ObtainMedia_AllowAbstract As Boolean = False
+        Protected Sub ObtainMedia(ByVal n As EContainer, ByVal PostID As String, Optional ByVal SpecialFolder As String = Nothing,
+                                  Optional ByVal DateObj As String = Nothing, Optional ByVal InitialType As Integer = -1,
+                                  Optional ByVal PostOriginUrl As String = Nothing)
             Try
+                Dim wrongData As Predicate(Of Sizes) = Function(_ss) _ss.HasError Or _ss.Data.IsEmptyString
                 Dim img As Predicate(Of EContainer) = Function(_img) Not _img.Name.IsEmptyString AndAlso _img.Name.StartsWith("image_versions") AndAlso _img.Count > 0
                 Dim vid As Predicate(Of EContainer) = Function(_vid) Not _vid.Name.IsEmptyString AndAlso _vid.Name.StartsWith("video_versions") AndAlso _vid.Count > 0
                 Dim ss As Func(Of EContainer, Sizes) = Function(_ss) New Sizes(_ss.Value("width"), _ss.Value("url"))
+                Dim ssVid As Func(Of EContainer, Sizes) = ss
+                Dim ssPic As Func(Of EContainer, Sizes) = ss
                 Dim mDate As Func(Of EContainer, String) = Function(ByVal elem As EContainer) As String
                                                                If Not DateObj.IsEmptyString Then Return DateObj
                                                                If elem.Contains("taken_at") Then
@@ -731,28 +764,41 @@ Namespace API.Instagram
                                                                    End If
                                                                End If
                                                            End Function
+                If Not ObtainMedia_SizeFuncVid Is Nothing Then ssVid = ObtainMedia_SizeFuncVid
+                If Not ObtainMedia_SizeFuncPic Is Nothing Then ssPic = ObtainMedia_SizeFuncPic
                 If n.Count > 0 Then
                     Dim l As New List(Of Sizes)
                     Dim d As EContainer
                     Dim t%
+                    Dim abstractDecision As Boolean = False
                     '8 - gallery
                     '2 - one video
                     '1 - one picture
                     t = n.Value("media_type").FromXML(Of Integer)(-1)
+                    If t = -1 And InitialType = 8 And ObtainMedia_AllowAbstract Then
+                        If n.Contains(vid) Then
+                            t = 2
+                            abstractDecision = True
+                        ElseIf n.Contains(img) Then
+                            t = 1
+                            abstractDecision = True
+                        End If
+                    End If
                     If t >= 0 Then
                         Select Case t
                             Case 1
                                 If n.Contains(img) Then
-                                    t = n.Value("media_type").FromXML(Of Integer)(-1)
+                                    If Not abstractDecision Then t = n.Value("media_type").FromXML(Of Integer)(-1)
                                     DateObj = mDate(n)
                                     If t >= 0 Then
                                         With n.ItemF({img, "candidates"}).XmlIfNothing
                                             If .Count > 0 Then
                                                 l.Clear()
-                                                l.ListAddList(.Select(ss), LNC)
+                                                l.ListAddList(.Select(ssPic), LNC)
+                                                If l.Count > 0 Then l.RemoveAll(wrongData)
                                                 If l.Count > 0 Then
                                                     l.Sort()
-                                                    _TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, l.First.Data, PostID, DateObj, SpecialFolder), LNC)
+                                                    _TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, l.First.Data, PostID, DateObj, SpecialFolder, PostOriginUrl), LNC)
                                                     l.Clear()
                                                 End If
                                             End If
@@ -765,10 +811,11 @@ Namespace API.Instagram
                                     With n.ItemF({vid}).XmlIfNothing
                                         If .Count > 0 Then
                                             l.Clear()
-                                            l.ListAddList(.Select(ss), LNC)
+                                            l.ListAddList(.Select(ssVid), LNC)
+                                            If l.Count > 0 Then l.RemoveAll(wrongData)
                                             If l.Count > 0 Then
                                                 l.Sort()
-                                                _TempMediaList.ListAddValue(MediaFromData(UTypes.Video, l.First.Data, PostID, DateObj, SpecialFolder), LNC)
+                                                _TempMediaList.ListAddValue(MediaFromData(UTypes.Video, l.First.Data, PostID, DateObj, SpecialFolder, PostOriginUrl), LNC)
                                                 l.Clear()
                                             End If
                                         End If
@@ -778,7 +825,7 @@ Namespace API.Instagram
                                 DateObj = mDate(n)
                                 With n("carousel_media").XmlIfNothing
                                     If .Count > 0 Then
-                                        For Each d In .Self : ObtainMedia(d, PostID, SpecialFolder, DateObj) : Next
+                                        For Each d In .Self : ObtainMedia(d, PostID, SpecialFolder, DateObj, 8, PostOriginUrl) : Next
                                     End If
                                 End With
                         End Select
@@ -939,6 +986,12 @@ Namespace API.Instagram
             DownloadContentDefault(Token)
         End Sub
 #End Region
+#Region "Erase"
+        Protected Overrides Sub EraseData_AdditionalDataFiles()
+            Dim f As SFile = MyFilePostsKV
+            If f.Exists Then f.Delete(SFO.File, SFODelete.DeleteToRecycleBin, EDP.ReturnValue)
+        End Sub
+#End Region
 #Region "Exceptions"
         ''' <exception cref="ExitException"></exception>
         ''' <inheritdoc cref="UserDataBase.ThrowAny(CancellationToken)"/>
@@ -996,9 +1049,9 @@ Namespace API.Instagram
 #End Region
 #Region "Create media"
         Private Function MediaFromData(ByVal t As UTypes, ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,
-                                       Optional ByVal SpecialFolder As String = Nothing) As UserMedia
+                                       Optional ByVal SpecialFolder As String = Nothing, Optional ByVal PostOriginUrl As String = Nothing) As UserMedia
             _URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
-            Dim m As New UserMedia(_URL, t) With {.Post = New UserPost With {.ID = PostID}}
+            Dim m As New UserMedia(_URL, t) With {.URL_BASE = PostOriginUrl.IfNullOrEmpty(_URL), .Post = New UserPost With {.ID = PostID}}
             If Not m.URL.IsEmptyString Then m.File = CStr(RegexReplace(m.URL, FilesPattern))
             If Not PostDate.IsEmptyString Then m.Post.Date = AConvert(Of Date)(PostDate, UnixDate32Provider, Nothing) Else m.Post.Date = Nothing
             m.SpecialFolder = SpecialFolder

SCrawler/API/JustForFans/UserData.vb

@@ -238,6 +238,8 @@ Namespace API.JustForFans
 
                                 _DownloadedPostsCount += 1
                                 _TempMediaList.ListAddList(post.GetUserMedia(FileSerialInstance), LNC)
+
+                                If _Limit > 0 And _DownloadedPostsCount >= _Limit Then Exit For
                             End If
                         Next
                     End If
@@ -250,11 +252,12 @@ Namespace API.JustForFans
         End Sub
         Private Sub GetUserID()
             Try
-                Dim r$, hash$, new_id$
+                Dim r$, hash$, new_id$, profilePic$
                 If ID.IsEmptyString Then
                     r = Responser.GetResponse($"https://justfor.fans/{Name}")
                     If Not r.IsEmptyString Then
                         hash = RegexReplace(r, RegexUser)
+                        profilePic = RegexReplace(r, RParams.DMS("<img class=.mainProfilePic..+?src=""([^""]+)", 1, EDP.ReturnValue))
                         If Not hash.IsEmptyString Then
                             r = Responser.GetResponse($"https://justfor.fans/ajax/getAssetCount.php?User={Name}&Ver={hash}")
                             If Not r.IsEmptyString Then
@@ -262,8 +265,14 @@ Namespace API.JustForFans
                                     If j.ListExists Then
                                         new_id = j.Value("UserID")
                                         If Not new_id.IsEmptyString Then
-                                            new_id = RegexReplace(new_id, RParams.DM("\D", 0, RegexReturn.Replace, CType(Function(input$) String.Empty, Func(Of String, String))))
-                                            If Not new_id.IsEmptyString Then ID = new_id : _ForceSaveUserInfo = True
+                                            new_id = RegexReplace(new_id, RParams.DM("\D", -1, RegexReturn.Replace,
+                                                                                     CType(Function(input$) String.Empty, Func(Of String, String)),
+                                                                                     String.Empty, EDP.ReturnValue))
+                                            If Not new_id.IsEmptyString Then
+                                                ID = new_id
+                                                _ForceSaveUserInfo = True
+                                                If Not profilePic.IsEmptyString Then GetWebFile(profilePic, $"{DownloadContentDefault_GetRootDir.CSFilePS}ProfilePic.jpg", EDP.None)
+                                            End If
                                         End If
                                     End If
                                 End Using

SCrawler/API/Mastodon/SiteSettings.vb

@@ -203,17 +203,21 @@ Namespace API.Mastodon
 #Region "UpdateServersList"
         Private Sub UpdateServersList()
             Try
-                Dim r$ = GetWebString("https://api.joinmastodon.org/servers?language=&category=&region=&ownership=&registrations=",, EDP.ThrowException)
-                If Not r.IsEmptyString Then
-                    Dim j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue)
-                    If If(j?.Count, 0) > 0 Then
-                        Domains.Domains.ListAddList(j.Select(Function(e) e.Value("domain")), LAP.NotContainsOnly, EDP.ReturnValue)
-                        Domains.Domains.Sort()
-                        Domains.Save()
-                        j.Dispose()
+                Using resp As New Responser With {
+                    .ProcessExceptionDecision = Function(rr, obj, e) If(rr.StatusCode = Net.HttpStatusCode.ServiceUnavailable,
+                                                                        EDP.ReturnValue, EDP.ThrowException)}
+                    Dim r$ = resp.GetResponse("https://api.joinmastodon.org/servers?language=&category=&region=&ownership=&registrations=")
+                    If Not r.IsEmptyString Then
+                        Dim j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue)
+                        If If(j?.Count, 0) > 0 Then
+                            Domains.Domains.ListAddList(j.Select(Function(e) e.Value("domain")), LAP.NotContainsOnly, EDP.ReturnValue)
+                            Domains.Domains.Sort()
+                            Domains.Save()
+                            j.Dispose()
+                        End If
+                        DomainsLastUpdateDate.Value = Now
                     End If
-                End If
-                DomainsLastUpdateDate.Value = Now
+                End Using
             Catch ex As Exception
                 ErrorsDescriber.Execute(EDP.SendToLog, ex, "[API.Mastodon.SiteSettings.UpdateServersList]")
             End Try

SCrawler/API/PornHub/Declarations.vb

@@ -14,6 +14,7 @@ Namespace API.PornHub
         Private ReadOnly UnicodeHexConverter As Func(Of String, String) = Function(Input) SymbolsConverter.UnicodeHex.Decode(Input, EDP.ReturnValue)
 #End Region
 #Region "Declarations video"
+        Friend ReadOnly RegexVideo_MediaDef As RParams = RParams.DMS("mediaDefinitions.:\s*(\[\{.+?\}\])", 1, RegexOptions.Singleline, EDP.ReturnValue)
         Friend ReadOnly RegexVideo_FlashVarsBlocks As RParams = RParams.DM("(?<=(flashvars_\['[nN]ext[vV]ideo'\]|flashvars_\d+[^ ]+? = media_\d+?);[\r\n]*?)(.+?)(?=;flashvars_\d+?)",
                                                                            0, RegexReturn.List, EDP.ReturnValue)
         Friend ReadOnly RegexVideo_FlashVars_Vars As RParams = RParams.DM("var ([\w\d]{10,})=("".+?)(?=(;|\Z))", 0, RegexReturn.List)

SCrawler/API/PornHub/UserData.vb

@@ -818,6 +818,15 @@ Namespace API.PornHub
 #End Region
 #Region "CreateVideoURL"
         Private Function CreateVideoURL(ByVal r As String) As String
+            If r.IsEmptyString Then
+                Return String.Empty
+            Else
+                Dim u$ = CreateVideoURL_FlashVars(r)
+                If u.IsEmptyString Then u = CreateVideoURL_MediaDef(r)
+                Return u
+            End If
+        End Function
+        Private Function CreateVideoURL_FlashVars(ByVal r As String) As String
             Try
                 Dim OutStr$ = String.Empty
                 Dim OutList As New List(Of String)
@@ -876,7 +885,26 @@ Namespace API.PornHub
                 MyMainLOG = $"{ToStringForLog()}: something is wrong when parsing flashvars.{vbCr}{regex_ex.Message}"
                 Return String.Empty
             Catch ex As Exception
-                Return ErrorsDescriber.Execute(EDP.SendToLog, ex, "[API.PornHub.UserData.CreateVideoURL]", String.Empty)
+                Return ErrorsDescriber.Execute(EDP.SendToLog, ex, "[API.PornHub.UserData.CreateVideoURL_FlashVars]", String.Empty)
+            End Try
+        End Function
+        Private Function CreateVideoURL_MediaDef(ByVal r As String) As String
+            Try
+                Dim result$ = String.Empty
+                If Not r.IsEmptyString Then
+                    Dim script$ = RegexReplace(r, RegexVideo_MediaDef)
+                    If Not script.IsEmptyString Then
+                        Using j As EContainer = JsonDocument.Parse(script)
+                            If j.ListExists Then
+                                Dim s As List(Of Sizes) = j.Select(Function(jj) New Sizes(jj.Value("quality"), jj.Value("videoUrl"))).ListWithRemove(Function(d) d.HasError Or d.Data.IsEmptyString)
+                                If s.ListExists Then s.Sort() : result = s(0).Data : s.Clear()
+                            End If
+                        End Using
+                    End If
+                End If
+                Return result
+            Catch ex As Exception
+                Return ErrorsDescriber.Execute(EDP.SendToLog, ex, "[API.PornHub.UserData.CreateVideoURL_MediaDef]", String.Empty)
             End Try
         End Function
 #End Region

SCrawler/API/Redgifs/UserData.vb

@@ -249,7 +249,7 @@ Namespace API.RedGifs
                                                           Optional ByVal EObj As Object = Nothing) As Integer
             Dim s As WebExceptionStatus = Responser.Status
             Dim sc As HttpStatusCode = Responser.StatusCode
-            If sc = HttpStatusCode.NotFound Or s = DataGone Then
+            If sc = HttpStatusCode.NotFound Or s = DataGone Or sc = DataGone Then
                 UserExists = False
             ElseIf sc = HttpStatusCode.Unauthorized Then
                 MyMainLOG = $"RedGifs credentials have expired [{CInt(sc)}]: {ToStringForLog()}"

SCrawler/API/ThreadsNet/SiteSettings.vb

@@ -0,0 +1,166 @@
+' Copyright (C) 2023  Andy https://github.com/AAndyProgram
+' This program is free software: you can redistribute it and/or modify
+' it under the terms of the GNU General Public License as published by
+' the Free Software Foundation, either version 3 of the License, or
+' (at your option) any later version.
+'
+' This program is distributed in the hope that it will be useful,
+' but WITHOUT ANY WARRANTY
+Imports SCrawler.API.Base
+Imports SCrawler.Plugin
+Imports SCrawler.Plugin.Attributes
+Imports PersonalUtilities.Tools.Web.Clients
+Imports PersonalUtilities.Tools.Web.Cookies
+Imports PersonalUtilities.Functions.RegularExpressions
+Imports IG = SCrawler.API.Instagram.SiteSettings
+Namespace API.ThreadsNet
+    <Manifest("AndyProgram_ThreadsNet"), SeparatedTasks(1)>
+    Friend Class SiteSettings : Inherits SiteSettingsBase
+#Region "Declarations"
+        Friend Overrides ReadOnly Property Icon As Icon
+            Get
+                Return My.Resources.SiteResources.ThreadsIcon_192
+            End Get
+        End Property
+        Private ReadOnly _Image As Image
+        Friend Overrides ReadOnly Property Image As Image
+            Get
+                Return _Image
+            End Get
+        End Property
+#Region "Authorization"
+        <PropertyOption(ControlText:="x-csrftoken", AllowNull:=False)>
+        Friend ReadOnly Property HH_CSRF_TOKEN As PropertyValue
+        <PropertyOption(ControlText:="x-ig-app-id", AllowNull:=False)>
+        Friend Property HH_IG_APP_ID As PropertyValue
+        <PropertyOption(ControlText:="x-asbd-id", AllowNull:=True)>
+        Friend Property HH_ASBD_ID As PropertyValue
+        <PropertyOption(ControlText:="sec-ch-ua", AllowNull:=True)>
+        Private Property HH_BROWSER As PropertyValue
+        <PropertyOption(ControlText:="sec-ch-ua-full", ControlToolTip:="sec-ch-ua-full-version-list", AllowNull:=True)>
+        Private Property HH_BROWSER_EXT As PropertyValue
+        <PropertyOption(ControlText:="sec-ch-ua-platform-ver", ControlToolTip:="sec-ch-ua-platform-version", AllowNull:=True, LeftOffset:=120)>
+        Private Property HH_PLATFORM As PropertyValue
+        <PropertyOption(ControlText:="UserAgent")>
+        Private ReadOnly Property HH_USER_AGENT As PropertyValue
+        Private Sub ChangeResponserFields(ByVal PropName As String, ByVal Value As Object)
+            If Not PropName.IsEmptyString Then
+                Dim f$ = String.Empty
+                Dim isUserAgent As Boolean = False
+                Select Case PropName
+                    Case NameOf(HH_IG_APP_ID) : f = IG.Header_IG_APP_ID
+                    Case NameOf(HH_ASBD_ID) : f = IG.Header_ASBD_ID
+                    Case NameOf(HH_CSRF_TOKEN) : f = IG.Header_CSRF_TOKEN
+                    Case NameOf(HH_BROWSER) : f = IG.Header_Browser
+                    Case NameOf(HH_BROWSER_EXT) : f = IG.Header_BrowserExt
+                    Case NameOf(HH_PLATFORM) : f = IG.Header_Platform
+                    Case NameOf(HH_USER_AGENT) : isUserAgent = True
+                End Select
+                If Not f.IsEmptyString Then
+                    Responser.Headers.Remove(f)
+                    If Not CStr(Value).IsEmptyString Then Responser.Headers.Add(f, CStr(Value))
+                ElseIf isUserAgent Then
+                    Responser.UserAgent = CStr(Value)
+                End If
+            End If
+        End Sub
+#End Region
+#End Region
+#Region "Initializer"
+        Friend Sub New()
+            MyBase.New("Threads", "threads.net")
+            _AllowUserAgentUpdate = False
+            _Image = My.Resources.SiteResources.ThreadsIcon_192.ToBitmap
+
+            Dim app_id$ = String.Empty
+            Dim token$ = String.Empty
+            Dim asbd$ = String.Empty
+            Dim browser$ = String.Empty
+            Dim browserExt$ = String.Empty
+            Dim platform$ = String.Empty
+            Dim useragent$ = String.Empty
+
+            With Responser
+                .Accept = "*/*"
+                'URGENT: remove after debug
+                .DeclaredError = EDP.SendToLog + EDP.ThrowException
+                If .UserAgentExists Then useragent = .UserAgent
+                With .Headers
+                    If .Count > 0 Then
+                        token = .Value(IG.Header_CSRF_TOKEN)
+                        app_id = .Value(IG.Header_IG_APP_ID)
+                        asbd = .Value(IG.Header_ASBD_ID)
+                        browser = .Value(IG.Header_Browser)
+                        browserExt = .Value(IG.Header_BrowserExt)
+                        platform = .Value(IG.Header_Platform)
+                    End If
+                    .Add("Authority", "www.threads.net")
+                    .Add("Origin", "https://www.threads.net")
+                    .Add("Upgrade-Insecure-Requests", 1)
+                    .Add("Sec-Ch-Ua-Model", "")
+                    .Add("Sec-Ch-Ua-Mobile", "?0")
+                    .Add("Sec-Ch-Ua-Platform", """Windows""")
+                    .Add("Sec-Fetch-Dest", "empty")
+                    .Add("Sec-Fetch-Mode", "cors")
+                    .Add("Sec-Fetch-Site", "same-origin")
+                    .Add("Sec-Fetch-User", "?1")
+                    .Add("x-fb-friendly-name", "BarcelonaProfileThreadsTabRefetchableQuery")
+                End With
+                .CookiesExtractMode = Responser.CookiesExtractModes.Any
+                .CookiesUpdateMode = CookieKeeper.UpdateModes.ReplaceByNameAll
+                .CookiesExtractedAutoSave = False
+                .Cookies.ChangedAllowInternalDrop = False
+                .Cookies.Changed = False
+            End With
+
+            HH_CSRF_TOKEN = New PropertyValue(token, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_CSRF_TOKEN), v))
+            HH_IG_APP_ID = New PropertyValue(app_id, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_IG_APP_ID), v))
+            HH_ASBD_ID = New PropertyValue(asbd, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_ASBD_ID), v))
+            HH_BROWSER = New PropertyValue(browser, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_BROWSER), v))
+            HH_BROWSER_EXT = New PropertyValue(browserExt, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_BROWSER_EXT), v))
+            HH_PLATFORM = New PropertyValue(platform, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_PLATFORM), v))
+            HH_USER_AGENT = New PropertyValue(useragent, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_USER_AGENT), v))
+
+            UrlPatternUser = "https://www.threads.net/@{0}"
+            UserRegex = RParams.DMS("threads.net/@([^/\?&]+)", 1)
+            ImageVideoContains = "threads.net"
+        End Sub
+#End Region
+#Region "UpdateResponserData"
+        Friend Sub UpdateResponserData(ByVal Resp As Responser)
+            With Responser.Cookies
+                Dim csrf$ = String.Empty
+                .Update(Resp.Cookies)
+                If .Changed Then
+                    Responser.SaveCookies()
+                    .Changed = False
+                    csrf = If(.FirstOrDefault(Function(c) c.Name.StringToLower = IG.Header_CSRF_TOKEN_COOKIE)?.Value, String.Empty)
+                End If
+                If Not csrf.IsEmptyString AndAlso Not AEquals(Of String)(csrf, HH_CSRF_TOKEN.Value) Then HH_CSRF_TOKEN.Value = csrf
+            End With
+        End Sub
+#End Region
+#Region "GetInstance"
+        Friend Overrides Function GetInstance(ByVal What As ISiteSettings.Download) As IPluginContentProvider
+            Return New UserData
+        End Function
+#End Region
+#Region "BaseAuthExists, GetUserUrl, GetUserPostUrl"
+        Friend Overrides Function BaseAuthExists() As Boolean
+            Return Responser.CookiesExists And {HH_CSRF_TOKEN, HH_IG_APP_ID}.All(Function(v) ACheck(Of String)(v.Value))
+        End Function
+        Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String
+            Return String.Format(UrlPatternUser, DirectCast(User, UserData).NameTrue)
+        End Function
+        Friend Overrides Function GetUserPostUrl(ByVal User As UserDataBase, ByVal Media As UserMedia) As String
+            Try
+                Dim code$ = DirectCast(User, UserData).GetPostCodeById(Media.Post.ID)
+                Dim name$ = DirectCast(User, UserData).NameTrue
+                If Not code.IsEmptyString Then Return $"https://www.threads.net/@{name}/post/{code}/" Else Return String.Empty
+            Catch ex As Exception
+                Return ErrorsDescriber.Execute(EDP.SendToLog, ex, "Can't open user's post", String.Empty)
+            End Try
+        End Function
+#End Region
+    End Class
+End Namespace

SCrawler/API/ThreadsNet/UserData.vb

@@ -0,0 +1,290 @@
+' Copyright (C) 2023  Andy https://github.com/AAndyProgram
+' This program is free software: you can redistribute it and/or modify
+' it under the terms of the GNU General Public License as published by
+' the Free Software Foundation, either version 3 of the License, or
+' (at your option) any later version.
+'
+' This program is distributed in the hope that it will be useful,
+' but WITHOUT ANY WARRANTY
+Imports System.Threading
+Imports SCrawler.API.Base
+Imports SCrawler.API.YouTube.Objects
+Imports PersonalUtilities.Functions.XML
+Imports PersonalUtilities.Functions.RegularExpressions
+Imports PersonalUtilities.Tools.Web.Documents.JSON
+Imports PersonalUtilities.Tools.Web.Clients
+Imports PersonalUtilities.Tools.Web.Clients.EventArguments
+Imports IGS = SCrawler.API.Instagram.SiteSettings
+Namespace API.ThreadsNet
+    Friend Class UserData : Inherits Instagram.UserData
+#Region "Declarations"
+        Private Const Header_FB_LSD As String = "x-fb-lsd"
+        Private ReadOnly Property MySettings As SiteSettings
+            Get
+                Return HOST.Source
+            End Get
+        End Property
+        Private ReadOnly ObtainMedia_SizeFuncPic_RegexP As RParams = RParams.DMS("_p(\d+)x(\d+)", 1, EDP.ReturnValue)
+        Private ReadOnly ObtainMedia_SizeFuncPic_RegexS As RParams = RParams.DMS("_s(\d+)x(\d+)", 1, EDP.ReturnValue)
+        Private ReadOnly DefaultParser_ElemNode_Default() As Object = {"node", "thread_items", 0, "post"}
+        Private OPT_LSD As String = String.Empty
+        Private OPT_FB_DTSG As String = String.Empty
+        Private ReadOnly Property Valid As Boolean
+            Get
+                Return Not OPT_LSD.IsEmptyString And Not OPT_FB_DTSG.IsEmptyString And Not ID.IsEmptyString
+            End Get
+        End Property
+#End Region
+#Region "Loader"
+        Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)
+        End Sub
+#End Region
+#Region "Exchange"
+        Friend Overrides Function ExchangeOptionsGet() As Object
+            Return Nothing
+        End Function
+        Friend Overrides Sub ExchangeOptionsSet(ByVal Obj As Object)
+        End Sub
+#End Region
+#Region "Initializer"
+        Friend Sub New()
+            ObtainMedia_SizeFuncPic = Function(ByVal ss As EContainer) As Sizes
+                                          If ss.Value("url").IsEmptyString Then
+                                              Return New Sizes("----", "")
+                                          ElseIf Not ss.Value("width").IsEmptyString Then
+                                              Return New Sizes(ss.Value("height").IfNullOrEmpty(ss.Value("width")), ss.Value("url"))
+                                          Else
+                                              Dim rval$ = RegexReplace(ss.Value("url"), ObtainMedia_SizeFuncPic_RegexP)
+                                              If Not rval.IsEmptyString Then Return New Sizes(rval, ss.Value("url"))
+                                              rval = RegexReplace(ss.Value("url"), ObtainMedia_SizeFuncPic_RegexS)
+                                              If Not rval.IsEmptyString Then Return New Sizes(AConvert(Of Integer)(rval, 1) * -1, ss.Value("url"))
+                                              Return New Sizes(10000, ss.Value("url"))
+                                          End If
+                                      End Function
+            ObtainMedia_SizeFuncVid = Function(ss) If(ss.Value("url").IsEmptyString, New Sizes("----", ""), New Sizes(10000, ss.Value("url")))
+            ObtainMedia_AllowAbstract = True
+            DefaultParser_ElemNode = DefaultParser_ElemNode_Default
+            DefaultParser_PostUrlCreator = Function(post) $"https://www.threads.net/@{NameTrue}/post/{post.Code}"
+        End Sub
+#End Region
+#Region "Download functions"
+        Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
+            Dim errorFound As Boolean = False
+            Try
+                Responser.Method = "POST"
+                AddHandler Responser.ResponseReceived, AddressOf Responser_ResponseReceived
+                LoadSavePostsKV(True)
+                OPT_LSD = String.Empty
+                OPT_FB_DTSG = String.Empty
+                DownloadData(String.Empty, Token)
+            Catch ex As Exception
+                errorFound = True
+                Throw ex
+            Finally
+                Responser.Method = "POST"
+                UpdateResponser()
+                MySettings.UpdateResponserData(Responser)
+                If Not errorFound Then LoadSavePostsKV(False)
+            End Try
+        End Sub
+        Protected Overrides Sub UpdateResponser()
+            If Not Responser Is Nothing AndAlso Not Responser.Disposed Then
+                RemoveHandler Responser.ResponseReceived, AddressOf Responser_ResponseReceived
+            End If
+        End Sub
+        Protected Overrides Sub Responser_ResponseReceived(ByVal Sender As Object, ByVal e As WebDataResponse)
+            If e.CookiesExists Then
+                Dim csrf$ = If(e.Cookies.FirstOrDefault(Function(v) v.Name.StringToLower = IGS.Header_CSRF_TOKEN_COOKIE)?.Value, String.Empty)
+                If Not csrf.IsEmptyString AndAlso Not AEquals(Of String)(csrf, Responser.Headers.Value(IGS.Header_CSRF_TOKEN)) Then _
+                   Responser.Headers.Add(IGS.Header_CSRF_TOKEN, csrf)
+            End If
+        End Sub
+        Private Overloads Sub DownloadData(ByVal Cursor As String, ByVal Token As CancellationToken)
+            Const urlPattern$ = "https://www.threads.net/api/graphql?lsd={0}&variables={1}&doc_id=6371597506283707&fb_api_req_friendly_name=BarcelonaProfileThreadsTabRefetchableQuery&server_timestamps=true&fb_dtsg={2}"
+            Const var_init$ = """userID"":""{0}"""
+            Const var_cursor$ = """after"":""{1}"",""before"":null,""first"":25,""last"":null,""userID"":""{0}"",""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsFeedbackHubEnabledrelayprovider"":false"
+            Dim URL$ = String.Empty
+            Try
+                If Not Valid Then
+                    Dim idIsNull As Boolean = ID.IsEmptyString
+                    UpdateCredentials()
+                    If idIsNull And Not ID.IsEmptyString Then _ForceSaveUserInfo = True
+                End If
+                If Not Valid Then Throw New Plugin.ExitException("Some credentials are missing")
+
+                Responser.Method = "POST"
+                Responser.Referer = $"https://www.threads.net/@{NameTrue}"
+                Responser.Headers.Add(Header_FB_LSD, OPT_LSD)
+
+                Dim nextCursor$ = String.Empty
+                Dim dataFound As Boolean = False
+
+                Dim vars$
+                If Cursor.IsEmptyString Then
+                    vars = String.Format(var_init, ID)
+                Else
+                    vars = String.Format(var_cursor, ID, Cursor)
+                End If
+                vars = SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & vars & "}")
+
+                URL = String.Format(urlPattern, OPT_LSD, vars, SymbolsConverter.ASCII.EncodeSymbolsOnly(OPT_FB_DTSG))
+
+                Using j As EContainer = GetDocument(URL, Token)
+                    If j.ListExists Then
+                        With j({"data", "mediaData"})
+                            If .ListExists Then
+                                nextCursor = .Value({"page_info"}, "end_cursor")
+                                With .Item({"edges"})
+                                    If .ListExists Then dataFound = DefaultParser(.Self, Sections.Timeline, Token)
+                                End With
+                            End If
+                        End With
+                    End If
+                End Using
+
+                If dataFound And Not nextCursor.IsEmptyString Then DownloadData(nextCursor, Token)
+            Catch ex As Exception
+                ProcessException(ex, Token, $"data downloading error [{URL}]")
+            End Try
+        End Sub
+        Private Function GetDocument(ByVal URL As String, ByVal Token As CancellationToken, Optional ByVal Round As Integer = 0) As EContainer
+            Try
+                ThrowAny(Token)
+                If Round > 0 AndAlso Not UpdateCredentials() Then Throw New Exception("Failed to update credentials")
+                ThrowAny(Token)
+                Dim r$ = Responser.GetResponse(URL)
+                If Not r.IsEmptyString Then Return JsonDocument.Parse(r) Else Throw New Exception("Failed to get a response")
+            Catch ex As Exception
+                If Round = 0 Then
+                    Return GetDocument(URL, Token, Round + 1)
+                Else
+                    Throw ex
+                End If
+            End Try
+        End Function
+        Private Function UpdateCredentials(Optional ByVal e As ErrorsDescriber = Nothing) As Boolean
+            Dim URL$ = $"https://www.threads.net/@{NameTrue}"
+            OPT_LSD = String.Empty
+            OPT_FB_DTSG = String.Empty
+            Try
+                Responser.Method = "GET"
+                Responser.Referer = URL
+                Responser.Headers.Remove(Header_FB_LSD)
+                Dim r$ = Responser.GetResponse(URL,, EDP.SendToLog + EDP.ThrowException)
+                Dim rr As RParams
+                Dim tt$, ttVal$
+                If Not r.IsEmptyString Then
+                    rr = RParams.DM("\[\],{""token"":""(.*?)""},\d+\]", 0, RegexReturn.List, EDP.ReturnValue)
+                    Dim tokens As List(Of String) = RegexReplace(r, rr)
+                    If tokens.ListExists Then
+                        With rr
+                            .Match = Nothing
+                            .MatchSub = 1
+                            .WhatGet = RegexReturn.Value
+                        End With
+                        For Each tt In tokens
+                            If Not OPT_FB_DTSG.IsEmptyString And Not OPT_LSD.IsEmptyString Then
+                                Exit For
+                            Else
+                                ttVal = RegexReplace(tt, rr)
+                                If Not ttVal.IsEmptyString Then
+                                    If ttVal.Contains(":") Then
+                                        If OPT_FB_DTSG.IsEmptyString Then OPT_FB_DTSG = ttVal
+                                    Else
+                                        If OPT_LSD.IsEmptyString Then OPT_LSD = ttVal
+                                    End If
+                                End If
+                            End If
+                        Next
+                    End If
+                    If ID.IsEmptyString Then ID = RegexReplace(r, RParams.DMS("""props"":\{""user_id"":""(\d+)""\},", 1, EDP.ReturnValue))
+                End If
+                Return Valid
+            Catch ex As Exception
+                Dim notFound$ = String.Empty
+                If OPT_FB_DTSG.IsEmptyString Then notFound.StringAppend(Header_FB_LSD)
+                If OPT_LSD.IsEmptyString Then notFound.StringAppend("lsd")
+                If ID.IsEmptyString Then notFound.StringAppend("User ID")
+                LogError(ex, $"failed to update some{IIf(notFound.IsEmptyString, String.Empty, $" ({notFound})")} credentials", e)
+                Return False
+            End Try
+        End Function
+#End Region
+#Region "ReparseMissing"
+        Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken)
+            Const varsPattern$ = """postID"":""{0}"",""userID"":""{1}"",""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsFeedbackHubEnabledrelayprovider"":false"
+            'Const varsPattern$ = "{""postID"":""{0}"",""__relay_internal__pv__BarcelonaIsLoggedInrelayprovider"":true,""__relay_internal__pv__BarcelonaIsFeedbackHubEnabledrelayprovider"":false}"
+            Const urlPattern$ = "https://www.threads.net/api/graphql?lsd={0}&variables={1}&fb_api_req_friendly_name=BarcelonaPostPageQuery&server_timestamps=true&fb_dtsg={2}&doc_id=25460088156920903"
+            Dim rList As New List(Of Integer)
+            Dim URL$ = String.Empty
+            DefaultParser_ElemNode = Nothing
+            DefaultParser_IgnorePass = True
+            Try
+                If ContentMissingExists Then
+                    Responser.Method = "POST"
+                    Responser.Referer = $"https://www.threads.net/@{NameTrue}"
+                    If Not IsSingleObjectDownload AndAlso Not UpdateCredentials() Then Throw New Exception("Failed to update credentials")
+                    Dim m As UserMedia
+                    Dim vars$
+                    Dim j As EContainer
+                    ProgressPre.ChangeMax(_ContentList.Count)
+                    For i% = 0 To _ContentList.Count - 1
+                        ProgressPre.Perform()
+                        m = _ContentList(i)
+                        If m.State = UserMedia.States.Missing And Not m.Post.ID.IsEmptyString Then
+                            ThrowAny(Token)
+                            vars = SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(varsPattern, m.Post.ID.Split("_").FirstOrDefault, ID) & "}")
+                            URL = String.Format(urlPattern, OPT_LSD, vars, SymbolsConverter.ASCII.EncodeSymbolsOnly(OPT_FB_DTSG))
+
+                            j = GetDocument(URL, Token)
+                            If j.ListExists Then
+                                With j.ItemF({"data", "data", "edges", 0, "node", "thread_items", 0, "post"})
+                                    If .ListExists AndAlso DefaultParser({ .Self}, Sections.Timeline, Token) Then rList.Add(i)
+                                End With
+                                j.Dispose()
+                            End If
+                        End If
+                    Next
+                End If
+            Catch ex As Exception
+                ProcessException(ex, Token, $"ReparseMissing error [{URL}]")
+            Finally
+                DefaultParser_ElemNode = DefaultParser_ElemNode_Default
+                DefaultParser_IgnorePass = False
+                If rList.Count > 0 Then
+                    For i% = rList.Count - 1 To 0 Step -1 : _ContentList.RemoveAt(rList(i)) : Next
+                    rList.Clear()
+                End If
+            End Try
+        End Sub
+#End Region
+#Region "DownloadSingleObject"
+        Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)
+            Dim url$ = Data.URL_BASE.IfNullOrEmpty(Data.URL)
+            Dim postCode$ = RegexReplace(url, RParams.DMS("post/([^/\?&]+)", 1, EDP.ReturnValue))
+            If Not postCode.IsEmptyString Then
+                Dim postId$ = CodeToID(postCode)
+                If Not postId.IsEmptyString Then
+                    _NameTrue = MySettings.IsMyUser(url).UserName
+                    DefaultParser_PostUrlCreator = Function(post) url
+                    If Not _NameTrue.IsEmptyString AndAlso UpdateCredentials(EDP.ReturnValue) Then
+                        _ContentList.Add(New UserMedia(url) With {.State = UserMedia.States.Missing, .Post = postId})
+                        ReparseMissing(Token)
+                    End If
+                End If
+            End If
+        End Sub
+#End Region
+#Region "ThrowAny"
+        Friend Overrides Sub ThrowAny(ByVal Token As CancellationToken)
+            ThrowAnyImpl(Token)
+        End Sub
+#End Region
+#Region "DownloadingException"
+        Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False,
+                                                          Optional ByVal EObj As Object = Nothing) As Integer
+            Return 0
+        End Function
+#End Region
+    End Class
+End Namespace

SCrawler/API/UserDataBind.vb

@@ -291,7 +291,7 @@ Namespace API
         End Property
         Friend Overrides Property ScriptUse As Boolean
             Get
-                Return Count > 0 AndAlso Collections.Exists(Function(c) c.ScriptUse)
+                Return Count > 0 AndAlso Collections.All(Function(c) c.ScriptUse)
             End Get
             Set(ByVal u As Boolean)
                 If Count > 0 Then Collections.ForEach(Sub(ByVal c As IUserData)
@@ -499,20 +499,20 @@ Namespace API
         Friend Overloads Sub Add(ByVal _Item As IUserData) Implements ICollection(Of IUserData).Add
             With _Item
                 If .MoveFiles(CollectionName, CollectionPath) Then
-                    If Not _Item.IsVirtual And DataMerging Then DirectCast(.Self, UserDataBase).MergeData()
-                    Collections.Add(_Item)
+                    If Not .Self.IsVirtual And DataMerging Then DirectCast(.Self, UserDataBase).MergeData()
+                    ConsolidateLabels(.Self)
+                    ConsolidateScripts(.Self)
+                    ConsolidateColors(.Self)
+                    Collections.Add(.Self)
                     With Collections.Last
-                        If Count > 1 Then
-                            If _CollectionName.IsEmptyString Then _CollectionName = .CollectionName
-                            .Temporary = Temporary
-                            .Favorite = Favorite
-                            .ReadyForDownload = ReadyForDownload
-                            ConsolidateLabels(_Item)
-                            ConsolidateScripts()
-                            ConsolidateColors(_Item)
-                            .UpdateUserInformation()
-                        End If
-                        MainFrameObj.ImageHandler(_Item, False)
+
+                        If _CollectionName.IsEmptyString Then _CollectionName = .CollectionName
+                        .Temporary = Temporary
+                        .Favorite = Favorite
+                        .ReadyForDownload = ReadyForDownload
+                        .UpdateUserInformation()
+
+                        MainFrameObj.ImageHandler(.Self, False)
                         AddRemoveBttDeleteHandler(.Self, True)
                         AddHandler .Self.UserUpdated, AddressOf User_OnUserUpdated
                     End With
@@ -550,8 +550,12 @@ Namespace API
         Private Sub ConsolidateLabels(ByVal Destination As UserDataBase)
             UpdateLabels(If(Destination, Me), ListAddList(Nothing, Labels.ListWithRemove(SpecialLabels)), 1, True)
         End Sub
-        Private Sub ConsolidateScripts()
-            If Count > 1 AndAlso ScriptUse Then Collections.ForEach(Sub(c) c.ScriptUse = True)
+        Private Sub ConsolidateScripts(ByVal Destination As UserDataBase)
+            If Count > 0 AndAlso ScriptUse Then
+                Dim __scriptData$ = Collections(0).ScriptData
+                Destination.ScriptUse = True
+                If Collections.All(Function(c) c.ScriptData = __scriptData) Then Destination.ScriptData = __scriptData
+            End If
         End Sub
         Private Sub ConsolidateColors(ByVal Destination As UserDataBase)
             If Count > 0 And Not Destination.ForeColor.HasValue And Not Destination.BackColor.HasValue Then

SCrawler/Editors/UserCreatorForm.vb

@@ -120,6 +120,7 @@ Namespace Editors
         Private SpecialPathHandler As PathMoverHandler = Nothing
         Friend ReadOnly Property UserLabels As List(Of String)
         Private LabelsIncludeSpecial As Boolean = False
+        Private LabelsChanged As Boolean = False
 #End Region
 #Region "Initializers"
         ''' <summary>Create new user</summary>
@@ -327,6 +328,8 @@ Namespace Editors
                 FriendlyNameChanged = False
             Catch ex As Exception
                 MyDef.InvokeLoaderError(ex)
+            Finally
+                LabelsChanged = False
             End Try
         End Sub
         Private Sub UserCreatorForm_KeyDown(sender As Object, e As KeyEventArgs) Handles Me.KeyDown
@@ -367,7 +370,7 @@ Namespace Editors
                             End If
                         End If
 
-                        If Not .Labels.ListEquals(UserLabels) Then _
+                        If LabelsChanged Then _
                            UserDataBase.UpdateLabels(.Self, UserLabels, 1,
                                                      Not DirectCast(.Self, UserDataBase).SpecialLabels.ListExists OrElse
                                                      UserDataBase.UpdateLabelsKeepSpecial(1))
@@ -596,7 +599,7 @@ CloseForm:
         Private Sub TXT_LABELS_ActionOnButtonClick(ByVal Sender As ActionButton, ByVal e As EventArgs) Handles TXT_LABELS.ActionOnButtonClick
             Select Case Sender.DefaultButton
                 Case ADB.Open : ChangeLabels()
-                Case ADB.Clear : UserLabels.Clear()
+                Case ADB.Clear : UserLabels.Clear() : LabelsChanged = True
                 Case ADB.Refresh : UpdateSpecificLabels(False)
             End Select
         End Sub
@@ -784,8 +787,10 @@ CloseForm:
             Using fl As New LabelsForm(UserLabels)
                 fl.ShowDialog()
                 If fl.DialogResult = DialogResult.OK Then
+                    LabelsChanged = True
                     UserLabels.ListAddList(fl.LabelsList, LAP.NotContainsOnly, LAP.ClearBeforeAdd)
                     If UserLabels.ListExists Then
+                        UserLabels.Sort()
                         TXT_LABELS.Text = UserLabels.ListToString
                     Else
                         TXT_LABELS.Clear()

SCrawler/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2023.9.20.0")>
-<Assembly: AssemblyFileVersion("2023.9.20.0")>
+<Assembly: AssemblyVersion("2023.10.1.0")>
+<Assembly: AssemblyFileVersion("2023.10.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/PluginsEnvironment/Hosts/LogHost.vb

@@ -13,8 +13,8 @@ Namespace Plugin.Hosts
         End Sub
         Friend Sub Add(ByVal ex As Exception, ByVal Message As String,
                        Optional ByVal ShowMainMsg As Boolean = False, Optional ByVal ShowErrorMsg As Boolean = False,
-                       Optional ByVal SendInLog As Boolean = True) Implements ILogProvider.Add
-            ErrorsDescriber.Execute(New ErrorsDescriber(ShowMainMsg, ShowErrorMsg, SendInLog), ex, Message)
+                       Optional ByVal SendToLog As Boolean = True) Implements ILogProvider.Add
+            ErrorsDescriber.Execute(New ErrorsDescriber(ShowMainMsg, ShowErrorMsg, SendToLog), ex, Message)
         End Sub
     End Class
 End Namespace

SCrawler/PluginsEnvironment/Hosts/PluginHost.vb

@@ -79,6 +79,7 @@ Namespace Plugin.Hosts
                     New PluginHost(New API.Twitter.SiteSettings, _XML, GlobalPath, _Temp, _Imgs, _Vids),
                     New PluginHost(New API.Mastodon.SiteSettings, _XML, GlobalPath, _Temp, _Imgs, _Vids),
                     New PluginHost(New API.Instagram.SiteSettings(_XML, GlobalPath), _XML, GlobalPath, _Temp, _Imgs, _Vids),
+                    New PluginHost(New API.ThreadsNet.SiteSettings, _XML, GlobalPath, _Temp, _Imgs, _Vids),
                     New PluginHost(New API.RedGifs.SiteSettings, _XML, GlobalPath, _Temp, _Imgs, _Vids),
                     New PluginHost(New API.YouTube.SiteSettings, _XML, GlobalPath, _Temp, _Imgs, _Vids),
                     New PluginHost(New API.Pinterest.SiteSettings, _XML, GlobalPath, _Temp, _Imgs, _Vids),

SCrawler/SCrawler.vbproj

@@ -223,6 +223,8 @@
     <Compile Include="API\ThisVid\SiteSettings.vb" />
     <Compile Include="API\ThisVid\UserData.vb" />
     <Compile Include="API\ThisVid\UserExchangeOptions.vb" />
+    <Compile Include="API\ThreadsNet\SiteSettings.vb" />
+    <Compile Include="API\ThreadsNet\UserData.vb" />
     <Compile Include="API\TikTok\Declarations.vb" />
     <Compile Include="API\TikTok\SiteSettings.vb" />
     <Compile Include="API\TikTok\UserData.vb" />
@@ -709,6 +711,9 @@
   <ItemGroup>
     <None Include="Content\Icons\SiteIcons\JFFIcon_64.ico" />
   </ItemGroup>
+  <ItemGroup>
+    <None Include="Content\Icons\SiteIcons\ThreadsIcon_192.ico" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.VisualBasic.targets" />
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>

SCrawler/SiteResources.Designer.vb

@@ -264,6 +264,16 @@ Namespace My.Resources
             End Get
         End Property
         
+        '''<summary>
+        '''  Looks up a localized resource of type System.Drawing.Icon similar to (Icon).
+        '''</summary>
+        Friend Shared ReadOnly Property ThreadsIcon_192() As System.Drawing.Icon
+            Get
+                Dim obj As Object = ResourceManager.GetObject("ThreadsIcon_192", resourceCulture)
+                Return CType(obj,System.Drawing.Icon)
+            End Get
+        End Property
+        
         '''<summary>
         '''  Looks up a localized resource of type System.Drawing.Icon similar to (Icon).
         '''</summary>

SCrawler/SiteResources.resx

@@ -178,6 +178,9 @@
   <data name="ThisVidPic_16" type="System.Resources.ResXFileRef, System.Windows.Forms">
     <value>Content\Pictures\SitePictures\ThisVidPic_16.png;System.Drawing.Bitmap, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
   </data>
+  <data name="ThreadsIcon_192" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>Content\Icons\SiteIcons\ThreadsIcon_192.ico;System.Drawing.Icon, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
+  </data>
   <data name="TikTokIcon_32" type="System.Resources.ResXFileRef, System.Windows.Forms">
     <value>Content\Icons\SiteIcons\TikTokIcon_32.ico;System.Drawing.Icon, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
   </data>