geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

2023.12.13.0

Browse Source 
YT
Structures: add 'YouTubeChannelTab' enum
YouTubeFunctions: add 'StandardizeURL_Channel' function; update 'Info_GetUrlType', 'Parse' and 'Parse_Internal' functions (YouTubeChannelTab)
YouTubeSettings: add 'ChannelsDownload' property and 'YouTubeChannelTabConverter' converter for grid
Add 'ChannelTabsChooserForm'
VideoListForm: remove buttons 'BTT_ADD_NO_SHORTS' and 'BTT_ADD_SHORTS_ONLY'; update 'BTT_ADD_KeyClick' function

SCrawler
API.M3U8Base: add 'M3U8URL' structure; update 'Download' function with 'OnlyDownload' arg
API.ProfileSaved: add sorting of feed files
API.Xhamster: update M3U8 parser and 'Download' function for additional downloading ways; add 'ReencodeVideos' property (to the 'SiteSettings')
API.YouTube: update to updated 'Parse' function
This commit is contained in:
Andy
2023-12-13 19:48:50 +03:00
parent 64d6e6b28c
commit 1455a31879
23 changed files with 882 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
SCrawler.YouTube/Base/Structures.vb
View File

@@ -62,6 +62,14 @@ Namespace API.YouTube.Base
Channel = 2
PlayList = 3
End Enum
<Editor(GetType(EnumDropDownEditor), GetType(UITypeEditor)), Flags>
Public Enum YouTubeChannelTab As Integer
<EnumValue(IsNullValue:=True)>
All = 0
Videos = 1
Shorts = 3
Playlists = 10
End Enum
<Editor(GetType(EnumDropDownEditor), GetType(UITypeEditor))>
Public Enum Protocols As Integer
<EnumValue(ExcludeFromList:=True)>

85
SCrawler.YouTube/Base/YouTubeFunctions.vb
View File

@@ -43,11 +43,35 @@ Namespace API.YouTube.Base
Return URL
End Try
End Function
Public Shared Function StandardizeURL_Channel(ByVal URL As String, Optional ByVal Process As Boolean = True) As String
Try
Dim ct As YouTubeChannelTab = YouTubeChannelTab.All
Dim isMusic As Boolean = False
If Process AndAlso Info_GetUrlType(URL, isMusic,,,, ct) = YouTubeMediaType.Channel AndAlso Not isMusic Then
If Not ct = YouTubeChannelTab.All Then
Dim rValue$ = String.Empty
Select Case ct
Case YouTubeChannelTab.Videos : rValue = "/videos"
Case YouTubeChannelTab.Shorts : rValue = "/shorts"
Case YouTubeChannelTab.Playlists : rValue = "/playlists"
End Select
If Not rValue.IsEmptyString Then
Dim startIndx% = InStr(URL, rValue)
If startIndx > 0 Then URL = URL.Remove(startIndx - 1)
End If
End If
End If
Return URL
Catch ex As Exception
Return URL
End Try
End Function
Public Shared Function IsMyUrl(ByVal URL As String) As Boolean
Return Not Info_GetUrlType(URL) = YouTubeMediaType.Undefined
End Function
Public Shared Function Info_GetUrlType(ByVal URL As String, Optional ByRef IsMusic As Boolean = False, Optional ByRef IsShorts As Boolean = False,
Optional ByRef IsChannelUser As Boolean = False, Optional ByRef Id As String = Nothing) As YouTubeMediaType
Optional ByRef IsChannelUser As Boolean = False, Optional ByRef Id As String = Nothing,
Optional ByRef ChannelOptions As YouTubeChannelTab = YouTubeChannelTab.All) As YouTubeMediaType
If Not URL.IsEmptyString Then
IsMusic = URL.Contains("music.youtube.com")
IsChannelUser = False
@@ -60,7 +84,17 @@ Namespace API.YouTube.Base
Case "watch" : Return YouTubeMediaType.Single
Case "shorts" : IsShorts = True : Return YouTubeMediaType.Single
Case "playlist" : Return YouTubeMediaType.PlayList
Case UserChannelOption, "@" : IsChannelUser = data(2).ToLower = UserChannelOption : Return YouTubeMediaType.Channel
Case UserChannelOption, "@"
IsChannelUser = data(2).ToLower = UserChannelOption
If data.Count > 6 Then
Select Case data(6).StringToLower.StringTrimStart("/")
Case "videos" : ChannelOptions = YouTubeChannelTab.Videos
Case "shorts" : ChannelOptions = YouTubeChannelTab.Shorts
Case "playlists" : ChannelOptions = YouTubeChannelTab.Playlists
Case Else : ChannelOptions = YouTubeChannelTab.All
End Select
End If
Return YouTubeMediaType.Channel
End Select
End If
End If
@@ -82,27 +116,25 @@ Namespace API.YouTube.Base
''' <exception cref="InvalidOperationException"></exception>
Public Shared Function Parse(ByVal URL As String, Optional ByVal UseCookies As Boolean? = Nothing,
Optional ByVal Token As Threading.CancellationToken = Nothing, Optional ByVal Progress As IMyProgress = Nothing,
Optional ByVal GetDefault As Boolean? = Nothing, Optional ByVal GetShorts As Boolean? = Nothing,
Optional ByVal DateAfter As Date? = Nothing, Optional ByVal DateBefore As Date? = Nothing) As IYouTubeMediaContainer
Optional ByVal DateAfter As Date? = Nothing, Optional ByVal DateBefore As Date? = Nothing,
Optional ByVal ChannelOption As YouTubeChannelTab? = Nothing, Optional ByVal UrlAsIs As Boolean = False) As IYouTubeMediaContainer
If URL.IsEmptyString Then Throw New ArgumentNullException("URL", "URL cannot be null")
If Not MyYouTubeSettings.YTDLP.Value.Exists Then Throw New IO.FileNotFoundException("Path to 'yt-dlp.exe' not set or program not found at destination", MyYouTubeSettings.YTDLP.Value.ToString)
Dim urlOrig$ = URL
URL = RegexReplace(URL, TrueUrlRegEx)
If URL.IsEmptyString Then Throw New ArgumentNullException("URL", $"Can't get true URL from [{urlOrig}]")
Dim isMusic As Boolean = False, isShorts As Boolean = False
Dim objType As YouTubeMediaType = Info_GetUrlType(URL, isMusic, isShorts)
Dim channelTab As YouTubeChannelTab = YouTubeChannelTab.All
Dim objType As YouTubeMediaType = Info_GetUrlType(URL, isMusic, isShorts,,, channelTab)
If ChannelOption.HasValue Then channelTab = ChannelOption.Value
If Not objType = YouTubeMediaType.Undefined Then
Dim __GetDefault As Boolean = If(GetDefault, True)
Dim __GetShorts As Boolean = If(GetShorts, True)
If isMusic Then __GetShorts = False
Dim container As IYouTubeMediaContainer
Dim pattern$ = "%(channel_id)s_%(id)s_%(playlist_index)s"
Select Case objType
Case YouTubeMediaType.Single
__GetShorts = False
If isMusic Then container = New Track Else container = New Video
Case YouTubeMediaType.PlayList : container = New PlayList : pattern = "%(playlist_index)s_%(id)s" : __GetShorts = False
Case YouTubeMediaType.PlayList : container = New PlayList : pattern = "%(playlist_index)s_%(id)s"
Case YouTubeMediaType.Channel
container = New Channel
If isMusic Then pattern = "%(playlist_id)s/%(channel_id)s_%(id)s_%(playlist_index)s"
@@ -121,11 +153,11 @@ Namespace API.YouTube.Base
Dim useCookiesForce As Boolean = UseCookies.HasValue AndAlso UseCookies.Value AndAlso cookiesExists
If UseCookies.HasValue AndAlso UseCookies.Value Then
withCookieRequested = True
result = Parse_Internal(URL, pattern, _CachePathDefault, True, YouTubeCookieNetscapeFile, DateAfter, DateBefore, __GetDefault, __GetShorts)
result = Parse_Internal(URL, pattern, _CachePathDefault, True, YouTubeCookieNetscapeFile, DateAfter, DateBefore, objType, channelTab, isMusic, UrlAsIs)
End If
If Not result And Not withCookieRequested Then
If Not UseCookies.HasValue OrElse Not UseCookies.Value Then result = Parse_Internal(URL, pattern, _CachePathDefault, False, YouTubeCookieNetscapeFile, DateAfter, DateBefore, __GetDefault, __GetShorts)
If Not result And Not UseCookies.HasValue And cookiesExists Then result = Parse_Internal(URL, pattern, _CachePathDefault, True, YouTubeCookieNetscapeFile, DateAfter, DateBefore, __GetDefault, __GetShorts)
If Not UseCookies.HasValue OrElse Not UseCookies.Value Then result = Parse_Internal(URL, pattern, _CachePathDefault, False, YouTubeCookieNetscapeFile, DateAfter, DateBefore, objType, channelTab, isMusic, UrlAsIs)
If Not result And Not UseCookies.HasValue And cookiesExists Then result = Parse_Internal(URL, pattern, _CachePathDefault, True, YouTubeCookieNetscapeFile, DateAfter, DateBefore, objType, channelTab, isMusic, UrlAsIs)
End If
If result Then
@@ -139,21 +171,40 @@ Namespace API.YouTube.Base
Private Shared Function Parse_Internal(ByVal URL As String, ByVal OutputPattern As String, ByVal OutputPath As SFile,
ByVal UseCookies As Boolean, ByVal CookiesFile As SFile,
ByVal DateAfter As Date?, ByVal DateBefore As Date?,
ByVal GetDefault As Boolean, ByVal GetShorts As Boolean) As Boolean
ByVal ObjType As YouTubeMediaType, ByVal ChannelTab As YouTubeChannelTab,
ByVal IsMusic As Boolean, ByVal UrlAsIs As Boolean) As Boolean
Try
Dim command$ = "yt-dlp --write-info-json --skip-download"
command.StringAppend(GetCookiesCommand(UseCookies, CookiesFile), " ")
If DateAfter.HasValue Then command.StringAppend($"--dateafter {DateAfter.Value:yyyyMMdd}", " ")
If DateBefore.HasValue Then command.StringAppend($"--datebefore {DateBefore.Value:yyyyMMdd}", " ")
command.StringAppend("{0}" & $" -o ""{OutputPattern}""", " ")
'#If DEBUG Then
'Debug.WriteLine(String.Format(command, URL))
'#End If
Dim debugString As Func(Of String, String) = Function(ByVal input As String) As String
#If DEBUG Then
Debug.WriteLine(String.Format(command, URL))
Debug.WriteLine(input)
#End If
Return input
End Function
Using batch As New BatchExecutor(True)
With batch
.CommandPermanent = BatchExecutor.GetDirectoryCommand(MyYouTubeSettings.YTDLP.Value)
If GetDefault Then .Execute(String.Format(command, URL))
If GetShorts Then .Execute(String.Format(command, $"{URL.StringTrimEnd("/")}/shorts"))
If ObjType = YouTubeMediaType.Channel And Not IsMusic And Not UrlAsIs Then
Dim ct As List(Of YouTubeChannelTab) = EnumExtract(Of YouTubeChannelTab)(ChannelTab,, True).ListIfNothing
If ct.Count = 0 Then
.Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/videos")))
.Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/shorts")))
.Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/playlists")))
Else
If ct.Contains(YouTubeChannelTab.Videos) Then .Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/videos")))
If ct.Contains(YouTubeChannelTab.Shorts) Then .Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/shorts")))
If ct.Contains(YouTubeChannelTab.Playlists) Then .Execute(debugString(String.Format(command, $"{URL.StringTrimEnd("/")}/playlists")))
End If
Else
.Execute(debugString(String.Format(command, URL)))
End If
End With
End Using
Return SFile.GetFiles(OutputPath,, IO.SearchOption.AllDirectories, EDP.ReturnValue).Count > 0

32
SCrawler.YouTube/Base/YouTubeSettings.vb
View File

@@ -223,6 +223,38 @@ Namespace API.YouTube.Base
Description("Add some additional info to the program info if you need")>
Friend ReadOnly Property ProgramDescription As XMLValue(Of String)
#End Region
#Region "Defaults ChannelsDownload"
<Browsable(True), GridVisible, XMLVN({"Defaults", "Channels"}), Category("Defaults"), DisplayName("Default download tabs for channels"),
Description("Default download tabs for downloading channels"), TypeConverter(GetType(YouTubeChannelTabConverter))>
Public ReadOnly Property ChannelsDownload As XMLValue(Of YouTubeChannelTab)
Private Class YouTubeChannelTabConverter : Inherits TypeConverter
Public Overrides Function ConvertTo(ByVal Context As ITypeDescriptorContext, ByVal Culture As CultureInfo, ByVal Value As Object,
ByVal DestinationType As Type) As Object
If Not DestinationType Is Nothing Then
If DestinationType Is GetType(String) Then
If IsNothing(Value) Then
Return YouTubeChannelTab.All.ToString
Else
Dim v As List(Of YouTubeChannelTab) = EnumExtract(Of YouTubeChannelTab)(Value,,, EDP.ReturnValue).ListIfNothing
If v.ListExists Then
v.Sort()
Return v.ListToStringE(, New ANumbers.EnumToStringProvider(GetType(YouTubeChannelTab)))
Else
Return YouTubeChannelTab.All.ToString
End If
End If
Else
If IsNothing(Value) Then
Return YouTubeChannelTab.All
Else
Return Value
End If
End If
End If
Return MyBase.ConvertTo(Context, Culture, Value, DestinationType)
End Function
End Class
#End Region
#Region "Defaults Video"
<Browsable(True), GridVisible, XMLVN({"DefaultsVideo"}, "MKV"), Category("Defaults Video"), DisplayName("Default format"),
TypeConverter(GetType(FieldsTypeConverter)), GridStandardValuesProvider(NameOf(AvailableVideoFormats_Impl)),