2023.9.30.1

YouTube: add URL standardization; change the 'Browse' button handler in 'VideoOptionsForm'
API.UserDataBase: add 'TokenQueue' to main function exceptions

SCrawler.YouTube/Base/YouTubeFunctions.vb

@@ -20,6 +20,29 @@ Namespace API.YouTube.Base
         Public Const UrlTypePattern As String = "(?<=https?://[^/]*?youtube.com/)((@|[^\?/&]+))([/\?]{0,1}(list=|v=|)([^\?/&]*))(?=(\S+|\Z|))"
         Private Sub New()
         End Sub
+        Public Shared Function StandardizeURL(ByVal URL As String) As String
+            Try
+                Dim isMusic As Boolean = False, isShorts As Boolean = False
+                If Info_GetUrlType(URL, isMusic, isShorts) = YouTubeMediaType.Single Then
+                    If Not isMusic And Not isShorts Then
+                        Dim videoOptionRegex As RParams = RParams.DMS("[\?&]v=([^\?&]+)", 1, EDP.ReturnValue)
+                        Dim data As List(Of String) = RegexReplace(URL, RParams.DMS(UrlTypePattern, 0, RegexReturn.ListByMatch, EDP.ReturnValue))
+                        Dim val$ = String.Empty
+                        If data.ListExists Then
+                            For Each d$ In data
+                                val = RegexReplace(d, videoOptionRegex)
+                                If Not val.IsEmptyString Then Exit For
+                            Next
+                            data.Clear()
+                        End If
+                        If Not val.IsEmptyString Then Return $"https://www.youtube.com/watch?v={val}"
+                    End If
+                End If
+                Return URL
+            Catch ex As Exception
+                Return URL
+            End Try
+        End Function
         Public Shared Function IsMyUrl(ByVal URL As String) As Boolean
             Return Not Info_GetUrlType(URL) = YouTubeMediaType.Undefined
         End Function

SCrawler.YouTube/Base/YouTubeSettings.vb

@@ -133,6 +133,9 @@ Namespace API.YouTube.Base
         End Property
 #End Region
 #Region "Defaults"
+        <Browsable(True), GridVisible, XMLVN({"Defaults"}, True), Category("Defaults"), DisplayName("Standardize URLs"),
+            Description("Standardize URLs by eliminating unwanted strings. Default: true.")>
+        Public ReadOnly Property StandardizeURLs As XMLValue(Of Boolean)
         <Browsable(True), GridVisible, XMLVN({"Defaults"}), Category("Defaults"), DisplayName("Replace modification date"),
             Description("Set the file date to the date the video was added (website) (if available). Default: false.")>
         Public ReadOnly Property ReplaceModificationDate As XMLValue(Of Boolean)

SCrawler.YouTube/Controls/VideoOptionsForm.vb

@@ -433,7 +433,7 @@ Namespace API.YouTube.Controls
         End Sub
 #End Region
 #Region "Footer"
-        Private Sub BTT_BROWSE_MouseClick(sender As Object, e As MouseEventArgs) Handles BTT_BROWSE.MouseClick
+        Private Sub BTT_BROWSE_MouseDown(sender As Object, e As MouseEventArgs) Handles BTT_BROWSE.MouseDown
             Dim f As SFile
 #Disable Warning BC40000
             If MyContainer.HasElements Then

SCrawler.YouTube/Downloader/VideoListForm.vb

@@ -247,6 +247,8 @@ Namespace DownloadObjects.STDownloader
                 If e.Control Then useCookies = True
                 Dim useCookiesParse As Boolean? = Nothing
                 If useCookies Then useCookiesParse = True
+                Dim standardizeUrls As Boolean = MyYouTubeSettings.StandardizeURLs
+                Dim standardize As Func(Of String, String) = Function(input) If(standardizeUrls, YouTubeFunctions.StandardizeURL(input), input)
 
                 Dim c As IYouTubeMediaContainer = Nothing
                 Dim url$ = String.Empty
@@ -264,7 +266,7 @@ Namespace DownloadObjects.STDownloader
                                     pForm.SetInitialValues(.Count, "Parsing playlists...")
                                     Dim containers As New List(Of IYouTubeMediaContainer)
                                     For Each u$ In .Self
-                                        containers.Add(YouTubeFunctions.Parse(u, useCookiesParse, pForm.Token, pForm.MyProgress, True, False))
+                                        containers.Add(YouTubeFunctions.Parse(standardize(u), useCookiesParse, pForm.Token, pForm.MyProgress, True, False))
                                         pForm.NextPlaylist()
                                         pForm.MyProgress.Perform()
                                     Next
@@ -295,7 +297,7 @@ Namespace DownloadObjects.STDownloader
                         pForm = New ParsingProgressForm
                         pForm.Show(Me)
                         pForm.SetInitialValues(1, "Parsing data...")
-                        c = YouTubeFunctions.Parse(url, useCookiesParse, pForm.Token, pForm.MyProgress, GetDefault, GetShorts)
+                        c = YouTubeFunctions.Parse(standardize(url), useCookiesParse, pForm.Token, pForm.MyProgress, GetDefault, GetShorts)
                         pForm.Dispose()
                     End If
                     If Not c Is Nothing Then

SCrawler/API/Base/UserDataBase.vb

@@ -1226,7 +1226,7 @@ BlockNullPicture:
                 End If
                 ThrowIfDisposed()
                 If Not _PictureExists Or _EnvirInvokeUserUpdated Then OnUserUpdated()
-            Catch oex As OperationCanceledException When Token.IsCancellationRequested Or TokenPersonal.IsCancellationRequested
+            Catch oex As OperationCanceledException When Token.IsCancellationRequested Or TokenPersonal.IsCancellationRequested Or TokenQueue.IsCancellationRequested
                 MyMainLOG = $"{ToStringForLog()}: downloading canceled"
                 Canceled = True
             Catch exit_ex As ExitException