2024.5.4.0

YT
remove dots from the end of the file name; add a setting to remove specific characters

SCrawler
API.Instagram: simplify the 'Connection closed' error
API.Reddit: update token refresh request; add 'BearerTokenUseCurl' hidden property
API.Threads: fix frong header name ('dnt'); update 'UpdateCredentials' function
AutoDownloader: change 'IndexOutOfRangeException' to 'Exception' in the 'Download' function
TDownloader: fix 'FilesUpdatePendingUsers' function (freeze)
UserSearchForm: add 'FriendlyName' to search results

Changelog.md

@@ -1,3 +1,18 @@
+# 2024.5.4.0
+
+*2024-05-04*
+
+- Added
+  - YouTube (standalone app): setting to remove specific characters (`Defaults` - `Remove characters`)
+  - Instagram: simplify the `Connection closed` error
+  - Users search: add 'FriendlyName' to search results
+- Fixed
+  - YouTube (standalone app): incorrect download processing when the file name ends with a dot (Issue #188)
+  - The program is freezes when editing users in some cases
+  - Sites
+    - Reddit: token update error
+    - Threads: unable to obtain credentials (ID)
+
 # 2024.4.26.0
 
 *2024-04-26*

SCrawler.YouTube/Base/YouTubeSettings.vb

@@ -243,6 +243,9 @@ Namespace API.YouTube.Base
         <Browsable(True), GridVisible(False), XMLVN({"Defaults"}), Category("Defaults"), DisplayName("Program description"),
             Description("Add some additional info to the program info if you need")>
         Friend ReadOnly Property ProgramDescription As XMLValue(Of String)
+        <Browsable(True), GridVisible(False), XMLVN({"Defaults"}, "%"""), Category("Defaults"), DisplayName("Remove characters"),
+            Description("Remove specific characters from a file name")>
+        Friend ReadOnly Property FileRemoveCharacters As XMLValue(Of String)
 #End Region
 #Region "Defaults ChannelsDownload"
         <Browsable(True), GridVisible, XMLVN({"Defaults", "Channels"}), Category("Defaults"), DisplayName("Default download tabs for channels"),

SCrawler.YouTube/Declarations.vb

@@ -59,8 +59,10 @@ Namespace API.YouTube
         Friend Function CleanFileName(ByVal f As SFile) As SFile
             If Not f.IsEmptyString And Not f.Name.IsEmptyString Then
                 Dim ff As SFile = f
-                ff.Name = ff.Name.StringRemoveWinForbiddenSymbols
+                ff.Name = ff.Name.StringRemoveWinForbiddenSymbols.StringTrim
-                If Not ff.Name.IsEmptyString Then ff.Name = ff.Name.Replace("%", String.Empty)
+                ff.Name = ff.Name.StringTrimEnd(".")
+                If Not ff.Name.IsEmptyString And Not MyYouTubeSettings.FileRemoveCharacters.IsEmptyString Then _
+                   ff.Name = ff.Name.StringReplaceSymbols(MyYouTubeSettings.FileRemoveCharacters.Value.AsList.ListCast(Of String).ToArray, String.Empty, EDP.ReturnValue)
                 If ff.Name.IsEmptyString Then ff.Name = "file"
                 Return ff
             Else

SCrawler.YouTube/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.14.0")>
+<Assembly: AssemblyVersion("2024.5.4.0")>
-<Assembly: AssemblyFileVersion("2024.4.14.0")>
+<Assembly: AssemblyFileVersion("2024.5.4.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.14.0")>
+<Assembly: AssemblyVersion("2024.5.4.0")>
-<Assembly: AssemblyFileVersion("2024.4.14.0")>
+<Assembly: AssemblyFileVersion("2024.5.4.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/API/Instagram/UserData.vb

@@ -1269,6 +1269,9 @@ NextPageBlock:
             ElseIf Responser.StatusCode = 560 Or Responser.StatusCode = HttpStatusCode.InternalServerError Then '560, 500
                 MySiteSettings.SkipUntilNextSession = True
                 Err5xx = Responser.StatusCode
+            ElseIf Responser.StatusCode = -1 And Responser.Status = -1 Then
+                MySiteSettings.SkipUntilNextSession = True
+                Err5xx = Responser.StatusCode
             Else
                 MyMainLOG = $"Something is wrong. Your credentials may have expired [{CInt(Responser.StatusCode)}/{CInt(Responser.Status)}]: {ToString()} [{s}]"
                 DisableSection(s)

SCrawler/API/Reddit/SiteSettings.vb

@@ -10,6 +10,7 @@ Imports SCrawler.API.Base
 Imports SCrawler.Plugin
 Imports SCrawler.Plugin.Attributes
 Imports PersonalUtilities.Tools.Web.Clients
+Imports PersonalUtilities.Tools.Web.Clients.Base
 Imports PersonalUtilities.Tools.Web.Documents.JSON
 Imports PersonalUtilities.Functions.XML
 Imports PersonalUtilities.Functions.RegularExpressions
@@ -34,6 +35,8 @@ Namespace API.Reddit
                                         "You can find different tokens in the responses. Make sure that bearer token belongs to Reddit and not RedGifs." & vbCr &
                                         "There is not need to add a token if you are not using cookies to download the timeline.", IsAuth:=True)>
         Friend ReadOnly Property BearerToken As PropertyValue
+        <PropertyOption(ControlText:="Use 'cUrl' to get a token", IsAuth:=True), PXML, PClonable, HiddenControl>
+        Private ReadOnly Property BearerTokenUseCurl As PropertyValue
 #Region "TokenUpdateInterval"
         <PropertyOption(ControlText:="Token refresh interval", ControlToolTip:="Interval (in minutes) to refresh the token",
                         AllowNull:=False, LeftOffset:=120, IsAuth:=True), PXML, PClonable>
@@ -82,6 +85,7 @@ Namespace API.Reddit
             ApiClientID = New PropertyValue(String.Empty, GetType(String))
             ApiClientSecret = New PropertyValue(String.Empty, GetType(String))
             BearerToken = New PropertyValue(token, GetType(String), Sub(v) Responser.Headers.Add(DeclaredNames.Header_Authorization, v))
+            BearerTokenUseCurl = New PropertyValue(True)
             TokenUpdateInterval = New PropertyValue(60 * 12)
             TokenUpdateIntervalProvider = New TokenRefreshIntervalProvider
             BearerTokenDateUpdate = New PropertyValue(Now.AddYears(-1))
@@ -269,14 +273,36 @@ Namespace API.Reddit
                     result = False
                     Dim r$ = String.Empty
                     Dim c% = 0
-                    Dim _found As Boolean
+                    Dim useCurl As Boolean = Settings.CurlFile.Exists And CBool(BearerTokenUseCurl.Value)
+                    Dim curlUsed As Boolean = useCurl
                     Do
                         c += 1
                         Using resp As New Responser With {
                             .Method = "POST",
-                            .ProcessExceptionDecision = Function(status, obj, ee) If(status.StatusCode = 429, EDP.ReturnValue, ee)
+                            .ProcessExceptionDecision = Function(ByVal status As IResponserStatus, ByVal nullArg As Object, ByVal currErr As ErrorsDescriber) As ErrorsDescriber
+                                                            If status.StatusCode = 429 Then
+                                                                useCurl = False
+                                                                Return EDP.ReturnValue
+                                                            ElseIf status.StatusCode = Net.HttpStatusCode.Forbidden And Not useCurl And Settings.CurlFile.Exists Then
+                                                                useCurl = True
+                                                                Return EDP.ReturnValue
+                                                            Else
+                                                                Return currErr
+                                                            End If
+                                                        End Function
                         }
                             With resp
+                                If useCurl Then
+                                    If Settings.CurlFile.Exists Then
+                                        curlUsed = True
+                                        .Mode = Responser.Modes.Curl
+                                        .CurlPath = Settings.CurlFile
+                                        .CurlArgumentsLeft = $"-d ""grant_type=password&username={UserName}&password={Password}"" --user ""{ClientID}:{ClientSecret}"""
+                                    Else
+                                        Throw New ArgumentNullException("cUrl file", "The path to the cUrl file is not specified")
+                                    End If
+                                Else
+                                    .Mode = Responser.Modes.Default
                                     With .PayLoadValues
                                         .Add("grant_type", "password")
                                         .Add("username", UserName)
@@ -285,13 +311,13 @@ Namespace API.Reddit
                                     .CredentialsUserName = ClientID
                                     .CredentialsPassword = ClientSecret
                                     .PreAuthenticate = True
+                                End If
                             End With
                             r = resp.GetResponse("https://www.reddit.com/api/v1/access_token",, EDP.ThrowException)
                         End Using
                         If Not r.IsEmptyString Then
                             Using j As EContainer = JsonDocument.Parse(r)
                                 If j.ListExists Then
-                                    _found = True
                                     Dim newToken$ = j.Value("access_token")
                                     If Not newToken.IsEmptyString Then
                                         BearerToken.Value = $"Bearer {newToken}"
@@ -302,7 +328,7 @@ Namespace API.Reddit
                                 End If
                             End Using
                         End If
-                    Loop While c < 5 And Not _found
+                    Loop While c < 5 And Not result
                 End If
                 Return result
             Catch ex As Exception

SCrawler/API/ThreadsNet/SiteSettings.vb

@@ -40,10 +40,10 @@ Namespace API.ThreadsNet
         Friend ReadOnly Property HH_ASBD_ID As PropertyValue
         <PropertyOption(ControlText:="sec-ch-ua", AllowNull:=True, IsAuth:=True,
                         InheritanceName:=SettingsCLS.HEADER_DEF_sec_ch_ua), ControlNumber(30), PClonable, PXML(OnlyForChecked:=True)>
-        Private ReadOnly Property HH_BROWSER As PropertyValue
+        Friend ReadOnly Property HH_BROWSER As PropertyValue
         <PropertyOption(ControlText:="sec-ch-ua-full", ControlToolTip:=SettingsCLS.HEADER_DEF_sec_ch_ua_full_version_list, AllowNull:=True, IsAuth:=True,
                         InheritanceName:=SettingsCLS.HEADER_DEF_sec_ch_ua_full_version_list), ControlNumber(40), PClonable, PXML(OnlyForChecked:=True)>
-        Private ReadOnly Property HH_BROWSER_EXT As PropertyValue
+        Friend ReadOnly Property HH_BROWSER_EXT As PropertyValue
         <PropertyOption(ControlText:="sec-ch-ua-platform-ver", ControlToolTip:=SettingsCLS.HEADER_DEF_sec_ch_ua_platform_version, AllowNull:=True, IsAuth:=True, LeftOffset:=135,
                         InheritanceName:=SettingsCLS.HEADER_DEF_sec_ch_ua_platform_version), ControlNumber(50), PClonable, PXML(OnlyForChecked:=True)>
         Friend ReadOnly Property HH_PLATFORM_VER As PropertyValue
@@ -127,9 +127,10 @@ Namespace API.ThreadsNet
                     .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "cors"))
                     .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "same-origin"))
                     .Add("Sec-Fetch-User", "?1")
-                    .Add("dht", 1)
+                    .Add("dnt", 1)
                     .Add("drp", 1)
                     .Add(Instagram.UserData.GQL_HEADER_FB_FRINDLY_NAME, "BarcelonaProfileThreadsTabRefetchableQuery")
+                    .Remove("dht")
                 End With
                 .CookiesExtractMode = Responser.CookiesExtractModes.Any
                 .CookiesUpdateMode = CookieKeeper.UpdateModes.ReplaceByNameAll

SCrawler/API/ThreadsNet/UserData.vb

@@ -164,16 +164,28 @@ Namespace API.ThreadsNet
                     .Method = "GET"
                     .Referer = URL
                     With .Headers
-                        .Remove(GQL_HEADER_FB_LSD)
+                        .Clear()
+                        .Add("dnt", 1)
+                        .Add("drp", 1)
+                        .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Authority, "www.threads.net"))
+                        .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Origin, "https://www.threads.net"))
+                        .Add("Sec-Ch-Ua-Model", "")
+                        .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaMobile, "?0"))
+                        .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform, """Windows"""))
                         .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, "document"))
                         .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "navigate"))
+                        .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "none"))
+                        .Add("Upgrade-Insecure-Requests", 1)
+                        .Add("Sec-Fetch-User", "?1")
+                        .Add(IGS.Header_Browser, MySettings.HH_BROWSER.Value)
+                        .Add(IGS.Header_BrowserExt, MySettings.HH_BROWSER_EXT.Value)
                     End With
                 End With
                 WaitTimer()
                 Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException)
                 If Not r.IsEmptyString Then
                     ParseTokens(r, 0)
-                    If ID.IsEmptyString Then ID = RegexReplace(r, RParams.DMS("""props"":\{""user_id"":""(\d+)""\},", 1, EDP.ReturnValue))
+                    If ID.IsEmptyString Then ID = RegexReplace(r, RParams.DMS("""props"":\{""user_id"":""(\d+)""", 1, EDP.ReturnValue))
                 End If
                 Return Valid
             Catch ex As Exception

SCrawler/Download/Automation/AutoDownloader.vb

@@ -572,7 +572,19 @@ Namespace DownloadObjects
                     With Downloader
                         .AutoDownloaderWorking = True
                         If .Downloaded.Count > 0 Then .Downloaded.RemoveAll(Function(u) Keys.Contains(u.Key)) : .InvokeDownloadsChangeEvent()
-                        Do : Try : doRound += 1 : .AddRange(users, True) : Exit Do : Catch iex As IndexOutOfRangeException : Thread.Sleep(200) : End Try : Loop While doRound < doLim
+                        Do
+                            Try
+                                doRound += 1
+                                .AddRange(users, True)
+                                Exit Do
+                            Catch iex As Exception
+                                If doRound = doLim Then
+                                    Throw iex
+                                Else
+                                    Thread.Sleep(200)
+                                End If
+                            End Try
+                        Loop While doRound <= doLim
                         While .Working Or .Count > 0 : notify.Invoke() : Thread.Sleep(200) : End While
                         .AutoDownloaderWorking = False
                         notify.Invoke
@@ -586,7 +598,7 @@ Namespace DownloadObjects
                     End With
                 End If
             Catch ex As Exception
-                ErrorsDescriber.Execute(EDP.SendToLog, ex, "[AutoDownloader.Download]")
+                ErrorsDescriber.Execute(EDP.SendToLog, ex, $"[AutoDownloader.Download({Name})]")
             Finally
                 Keys.Clear()
                 LastDownloadDate = Now

SCrawler/Download/Feed/FeedSpecialCollection.vb

@@ -236,7 +236,7 @@ Namespace DownloadObjects
         Friend Sub UpdateUsers(ByVal InitialUser As UserInfo, ByVal NewUser As UserInfo)
             Try
                 Load()
-                If Count > 0 Then
+                If Count > 0 AndAlso Not UserInfo.ExactEquals(InitialUser, NewUser) Then
                     Feeds.ForEach(Sub(f) f.UpdateUsers(InitialUser, NewUser))
                     If Downloader.Files.Count > 0 Then
                         PendingUsersToUpdate.Add(New KeyValuePair(Of UserInfo, UserInfo)(InitialUser, NewUser))

SCrawler/Download/TDownloader.vb

@@ -191,9 +191,9 @@ Namespace DownloadObjects
         End Sub
         Private _FilesUpdating As Boolean = False
         Friend Sub FilesUpdatePendingUsers()
-            _FilesUpdating = True
             Try
                 If Files.Count > 0 Then
+                    _FilesUpdating = True
                     With Settings.Feeds
                         Dim pUsers As List(Of KeyValuePair(Of UserInfo, UserInfo))
                         Dim pendingUser As KeyValuePair(Of UserInfo, UserInfo)
@@ -214,19 +214,21 @@ Namespace DownloadObjects
                                         Next
                                     End If
                                 End With
-                                If changed Then FilesSave()
+                                If changed Then _FilesUpdating = False : FilesSave()
                             Next
                             pUsers.Clear()
                         End While
                     End With
+                    _FilesUpdating = False
                 End If
             Catch aex As ArgumentOutOfRangeException
+                _FilesUpdating = False
             Catch iex As IndexOutOfRangeException
+                _FilesUpdating = False
             Catch ex As Exception
+                _FilesUpdating = False
                 ErrorsDescriber.Execute(EDP.SendToLog, ex, "[TDownloader.FilesUpdatePendingUsers]")
                 MainFrameObj.UpdateLogButton()
-            Finally
-                _FilesUpdating = False
             End Try
         End Sub
         Friend Sub ClearSessions()

SCrawler/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.26.0")>
+<Assembly: AssemblyVersion("2024.5.4.0")>
-<Assembly: AssemblyFileVersion("2024.4.26.0")>
+<Assembly: AssemblyFileVersion("2024.5.4.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/UserSearchForm.vb

@@ -31,6 +31,7 @@ Friend Class UserSearchForm
             Else
                 If User.IncludedInCollection Then Text &= $"[{User.CollectionName}] "
                 Text &= $"[{User.Site}] [{User.Name}]"
+                If Not User.FriendlyName.IsEmptyString Then Text &= $" ({User.FriendlyName})"
             End If
         End Sub
         Private Function CompareTo(ByVal Other As SearchResult) As Integer Implements IComparable(Of SearchResult).CompareTo