mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-14 15:52:18 +00:00
2023.12.13.0
YT Structures: add 'YouTubeChannelTab' enum YouTubeFunctions: add 'StandardizeURL_Channel' function; update 'Info_GetUrlType', 'Parse' and 'Parse_Internal' functions (YouTubeChannelTab) YouTubeSettings: add 'ChannelsDownload' property and 'YouTubeChannelTabConverter' converter for grid Add 'ChannelTabsChooserForm' VideoListForm: remove buttons 'BTT_ADD_NO_SHORTS' and 'BTT_ADD_SHORTS_ONLY'; update 'BTT_ADD_KeyClick' function SCrawler API.M3U8Base: add 'M3U8URL' structure; update 'Download' function with 'OnlyDownload' arg API.ProfileSaved: add sorting of feed files API.Xhamster: update M3U8 parser and 'Download' function for additional downloading ways; add 'ReencodeVideos' property (to the 'SiteSettings') API.YouTube: update to updated 'Parse' function
This commit is contained in:
Andy
@@ -43,11 +43,35 @@ Namespace API.YouTube.Base
|
||||
Return URL
|
||||
End Try
|
||||
End Function
|
||||
Public Shared Function StandardizeURL_Channel(ByVal URL As String, Optional ByVal Process As Boolean = True) As String
|
||||
Try
|
||||
Dim ct As YouTubeChannelTab = YouTubeChannelTab.All
|
||||
Dim isMusic As Boolean = False
|
||||
If Process AndAlso Info_GetUrlType(URL, isMusic,,,, ct) = YouTubeMediaType.Channel AndAlso Not isMusic Then
|
||||
If Not ct = YouTubeChannelTab.All Then
|
||||
Dim rValue$ = String.Empty
|
||||
Select Case ct
|
||||
Case YouTubeChannelTab.Videos : rValue = "/videos"
|
||||
Case YouTubeChannelTab.Shorts : rValue = "/shorts"
|
||||
Case YouTubeChannelTab.Playlists : rValue = "/playlists"
|
||||
End Select
|
||||
If Not rValue.IsEmptyString Then
|
||||
Dim startIndx% = InStr(URL, rValue)
|
||||
If startIndx > 0 Then URL = URL.Remove(startIndx - 1)
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
Return URL
|
||||
Catch ex As Exception
|
||||
Return URL
|
||||
End Try
|
||||
End Function
|
||||
Public Shared Function IsMyUrl(ByVal URL As String) As Boolean
|
||||
Return Not Info_GetUrlType(URL) = YouTubeMediaType.Undefined
|
||||
End Function
|
||||
Public Shared Function Info_GetUrlType(ByVal URL As String, Optional ByRef IsMusic As Boolean = False, Optional ByRef IsShorts As Boolean = False,
|
||||
Optional ByRef IsChannelUser As Boolean = False, Optional ByRef Id As String = Nothing) As YouTubeMediaType
|
||||
Optional ByRef IsChannelUser As Boolean = False, Optional ByRef Id As String = Nothing,
|
||||
Optional ByRef ChannelOptions As YouTubeChannelTab = YouTubeChannelTab.All) As YouTubeMediaType
|
||||
If Not URL.IsEmptyString Then
|
||||
IsMusic = URL.Contains("music.youtube.com")
|
||||
IsChannelUser = False
|
||||
@@ -60,7 +84,17 @@ Namespace API.YouTube.Base
|
||||
Case "watch" : Return YouTubeMediaType.Single
|
||||
Case "shorts" : IsShorts = True : Return YouTubeMediaType.Single
|
||||
Case "playlist" : Return YouTubeMediaType.PlayList
|
||||
Case UserChannelOption, "@" : IsChannelUser = data(2).ToLower = UserChannelOption : Return YouTubeMediaType.Channel
|
||||
Case UserChannelOption, "@"
|
||||
IsChannelUser = data(2).ToLower = UserChannelOption
|
||||
If data.Count > 6 Then
|
||||
Select Case data(6).StringToLower.StringTrimStart("/")
|
||||
Case "videos" : ChannelOptions = YouTubeChannelTab.Videos
|
||||
Case "shorts" : ChannelOptions = YouTubeChannelTab.Shorts
|
||||
Case "playlists" : ChannelOptions = YouTubeChannelTab.Playlists
|
||||
Case Else : ChannelOptions = YouTubeChannelTab.All
|
||||
End Select
|
||||
End If
|
||||
Return YouTubeMediaType.Channel
|
||||
End Select
|
||||
End If
|
||||
End If
|
||||
@@ -82,27 +116,25 @@ Namespace API.YouTube.Base
|
||||
''' <exception cref="InvalidOperationException"></exception>
|
||||
Public Shared Function Parse(ByVal URL As String, Optional ByVal UseCookies As Boolean? = Nothing,
|
||||
Optional ByVal Token As Threading.CancellationToken = Nothing, Optional ByVal Progress As IMyProgress = Nothing,
|
||||
Optional ByVal GetDefault As Boolean? = Nothing, Optional ByVal GetShorts As Boolean? = Nothing,
|
||||
Optional ByVal DateAfter As Date? = Nothing, Optional ByVal DateBefore As Date? = Nothing) As IYouTubeMediaContainer
|
||||
Optional ByVal DateAfter As Date? = Nothing, Optional ByVal DateBefore As Date? = Nothing,
|
||||
Optional ByVal ChannelOption As YouTubeChannelTab? = Nothing, Optional ByVal UrlAsIs As Boolean = False) As IYouTubeMediaContainer
|
||||
If URL.IsEmptyString Then Throw New ArgumentNullException("URL", "URL cannot be null")
|
||||
If Not MyYouTubeSettings.YTDLP.Value.Exists Then Throw New IO.FileNotFoundException("Path to 'yt-dlp.exe' not set or program not found at destination", MyYouTubeSettings.YTDLP.Value.ToString)
|
||||
Dim urlOrig$ = URL
|
||||
URL = RegexReplace(URL, TrueUrlRegEx)
|
||||
If URL.IsEmptyString Then Throw New ArgumentNullException("URL", $"Can't get true URL from [{urlOrig}]")
|
||||
Dim isMusic As Boolean = False, isShorts As Boolean = False
|
||||
Dim objType As YouTubeMediaType = Info_GetUrlType(URL, isMusic, isShorts)
|
||||
Dim channelTab As YouTubeChannelTab = YouTubeChannelTab.All
|
||||
Dim objType As YouTubeMediaType = Info_GetUrlType(URL, isMusic, isShorts,,, channelTab)
|
||||
If ChannelOption.HasValue Then channelTab = ChannelOption.Value
|
||||
If Not objType = YouTubeMediaType.Undefined Then
|
||||
Dim __GetDefault As Boolean = If(GetDefault, True)
|
||||
Dim __GetShorts As Boolean = If(GetShorts, True)
|
||||
If isMusic Then __GetShorts = False
|
||||
Dim container As IYouTubeMediaContainer
|
||||
Dim pattern$ = "%(channel_id)s_%(id)s_%(playlist_index)s"
|
||||
|
||||
Select Case objType
|
||||
Case YouTubeMediaType.Single
|
||||
__GetShorts = False
|
||||
If isMusic Then container = New Track Else container = New Video
|
||||
Case YouTubeMediaType.PlayList : container = New PlayList : pattern = "%(playlist_index)s_%(id)s" : __GetShorts = False
|
||||
Case YouTubeMediaType.PlayList : container = New PlayList : pattern = "%(playlist_index)s_%(id)s"
|
||||
Case YouTubeMediaType.Channel
|
||||
container = New Channel
|
||||
If isMusic Then pattern = "%(playlist_id)s/%(channel_id)s_%(id)s_%(playlist_index)s"
|
||||
@@ -121,11 +153,11 @@ Namespace API.YouTube.Base
|
||||
Dim useCookiesForce As Boolean = UseCookies.HasValue AndAlso UseCookies.Value AndAlso cookiesExists
|
||||
If UseCookies.HasValue AndAlso UseCookies.Value Then
|
||||
withCookieRequested = True
|
||||
result = Parse_Internal(URL, pattern, _CachePathDefault, True, YouTubeCookieNetscapeFile, DateAfter, DateBefore, __GetDefault, __GetShorts)
|
||||
result = Parse_Internal(URL, pattern, _CachePathDefault, True, YouTubeCookieNetscapeFile, DateAfter, DateBefore, objType, channelTab, isMusic, UrlAsIs)
|
||||
End If
|
||||
If Not result And Not withCookieRequested Then
|
||||
If Not UseCookies.HasValue OrElse Not UseCookies.Value Then result = Parse_Internal(URL, pattern, _CachePathDefault, False, YouTubeCookieNetscapeFile, DateAfter, DateBefore, __GetDefault, __GetShorts)
|
||||
If Not result And Not UseCookies.HasValue And cookiesExists Then result = Parse_Internal(URL, pattern, _CachePathDefault, True, YouTubeCookieNetscapeFile, DateAfter, DateBefore, __GetDefault, __GetShorts)
|
||||
If Not UseCookies.HasValue OrElse Not UseCookies.Value Then result = Parse_Internal(URL, pattern, _CachePathDefault, False, YouTubeCookieNetscapeFile, DateAfter, DateBefore, objType, channelTab, isMusic, UrlAsIs)
|
||||
If Not result And Not UseCookies.HasValue And cookiesExists Then result = Parse_Internal(URL, pattern, _CachePathDefault, True, YouTubeCookieNetscapeFile, DateAfter, DateBefore, objType, channelTab, isMusic, UrlAsIs)
|
||||
End If
|
||||
|
||||
If result Then
|
||||
@@ -139,21 +171,40 @@ Namespace API.YouTube.Base
|
||||
Private Shared Function Parse_Internal(ByVal URL As String, ByVal OutputPattern As String, ByVal OutputPath As SFile,
|
||||
ByVal UseCookies As Boolean, ByVal CookiesFile As SFile,
|
||||
ByVal DateAfter As Date?, ByVal DateBefore As Date?,
|
||||
ByVal GetDefault As Boolean, ByVal GetShorts As Boolean) As Boolean
|
||||
ByVal ObjType As YouTubeMediaType, ByVal ChannelTab As YouTubeChannelTab,
|
||||
ByVal IsMusic As Boolean, ByVal UrlAsIs As Boolean) As Boolean
|
||||
Try
|
||||
Dim command$ = "yt-dlp --write-info-json --skip-download"
|
||||
command.StringAppend(GetCookiesCommand(UseCookies, CookiesFile), " ")
|
||||
If DateAfter.HasValue Then command.StringAppend($"--dateafter {DateAfter.Value:yyyyMMdd}", " ")
|
||||
If DateBefore.HasValue Then command.StringAppend($"--datebefore {DateBefore.Value:yyyyMMdd}", " ")
|
||||
command.StringAppend("{0}" & $" -o ""{OutputPattern}""", " ")
|
||||
'#If DEBUG Then
|
||||
'Debug.WriteLine(String.Format(command, URL))
|
||||
'#End If
|
||||
Dim debugString As Func(Of String, String) = Function(ByVal input As String) As String
|
||||
#If DEBUG Then
|
||||
Debug.WriteLine(String.Format(command, URL))
|
||||
Debug.WriteLine(input)
|
||||
#End If
|
||||
Return input
|
||||
End Function
|
||||
Using batch As New BatchExecutor(True)
|
||||
With batch
|
||||
.CommandPermanent = BatchExecutor.GetDirectoryCommand(MyYouTubeSettings.YTDLP.Value)
|
||||
If GetDefault Then .Execute(String.Format(command, URL))
|
||||
If GetShorts Then .Execute(String.Format(command, $"{URL.StringTrimEnd("/")}/shorts"))
|
||||
If ObjType = YouTubeMediaType.Channel And Not IsMusic And Not UrlAsIs Then
|
||||
Dim ct As List(Of YouTubeChannelTab) = EnumExtract(Of YouTubeChannelTab)(ChannelTab,, True).ListIfNothing
|
||||
If ct.Count = 0 Then
|
||||
.Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/videos")))
|
||||
.Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/shorts")))
|
||||
.Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/playlists")))
|
||||
Else
|
||||
If ct.Contains(YouTubeChannelTab.Videos) Then .Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/videos")))
|
||||
If ct.Contains(YouTubeChannelTab.Shorts) Then .Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/shorts")))
|
||||
If ct.Contains(YouTubeChannelTab.Playlists) Then .Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/playlists")))
|
||||
End If
|
||||
Else
|
||||
.Execute(debugString(String.Format(command, URL)))
|
||||
End If
|
||||
End With
|
||||
End Using
|
||||
Return SFile.GetFiles(OutputPath,, IO.SearchOption.AllDirectories, EDP.ReturnValue).Count > 0
|
||||
|
||||
Reference in New Issue
Block a user