2023.12.13.0

YT
Structures: add 'YouTubeChannelTab' enum
YouTubeFunctions: add 'StandardizeURL_Channel' function; update 'Info_GetUrlType', 'Parse' and 'Parse_Internal' functions (YouTubeChannelTab)
YouTubeSettings: add 'ChannelsDownload' property and 'YouTubeChannelTabConverter' converter for grid
Add 'ChannelTabsChooserForm'
VideoListForm: remove buttons 'BTT_ADD_NO_SHORTS' and 'BTT_ADD_SHORTS_ONLY'; update 'BTT_ADD_KeyClick' function

SCrawler
API.M3U8Base: add 'M3U8URL' structure; update 'Download' function with 'OnlyDownload' arg
API.ProfileSaved: add sorting of feed files
API.Xhamster: update M3U8 parser and 'Download' function for additional downloading ways; add 'ReencodeVideos' property (to the 'SiteSettings')
API.YouTube: update to updated 'Parse' function
This commit is contained in:
Andy
2023-12-13 19:48:50 +03:00
parent 64d6e6b28c
commit 1455a31879
23 changed files with 882 additions and 332 deletions

View File

@@ -18,8 +18,30 @@ Namespace API.Base
Friend ReadOnly TsFilesRegEx As RParams = RParams.DM(".+?\.ts[^\r\n]*", 0, RegexReturn.List)
End Module
End Namespace
Friend Structure M3U8URL
Friend URL As String
Friend Extension As String
Friend Sub New(ByVal _URL As String, Optional ByVal _Extension As String = Nothing)
URL = _URL
Extension = _Extension
End Sub
Public Shared Widening Operator CType(ByVal URL As String) As M3U8URL
Return New M3U8URL(URL)
End Operator
Public Overrides Function Equals(ByVal Obj As Object) As Boolean
If Not IsNothing(Obj) Then
If TypeOf Obj Is M3U8URL Then
Return CType(Obj, M3U8URL).URL = URL
Else
Return CStr(Obj) = URL
End If
End If
Return False
End Function
End Structure
Friend NotInheritable Class M3U8Base
Friend Const TempCacheFolderName As String = "tmpCache"
Friend Const TempFilePrefix As String = "ConPart_"
Private Sub New()
End Sub
Friend Shared Function CreateUrl(ByVal Appender As String, ByVal File As String) As String
@@ -32,9 +54,17 @@ Namespace API.Base
Return $"{Appender.StringTrimEnd("/")}/{File}"
End If
End Function
Friend Shared Function Download(ByVal URLs As List(Of String), ByVal DestinationFile As SFile, Optional ByVal Responser As Responser = Nothing,
Optional ByVal Token As CancellationToken = Nothing, Optional ByVal Progress As MyProgress = Nothing,
Optional ByVal UsePreProgress As Boolean = True, Optional ByVal ExistingCache As CacheKeeper = Nothing) As SFile
Friend Overloads Shared Function Download(ByVal URLs As List(Of String), ByVal DestinationFile As SFile, Optional ByVal Responser As Responser = Nothing,
Optional ByVal Token As CancellationToken = Nothing, Optional ByVal Progress As MyProgress = Nothing,
Optional ByVal UsePreProgress As Boolean = True, Optional ByVal ExistingCache As CacheKeeper = Nothing,
Optional ByVal OnlyDownload As Boolean = False) As SFile
Return Download(URLs.ListCast(Of M3U8URL), DestinationFile, Responser, Token, Progress, UsePreProgress, ExistingCache, OnlyDownload)
End Function
Friend Overloads Shared Function Download(ByVal URLs As List(Of M3U8URL), ByVal DestinationFile As SFile, Optional ByVal Responser As Responser = Nothing,
Optional ByVal Token As CancellationToken = Nothing, Optional ByVal Progress As MyProgress = Nothing,
Optional ByVal UsePreProgress As Boolean = True, Optional ByVal ExistingCache As CacheKeeper = Nothing,
Optional ByVal OnlyDownload As Boolean = False) As SFile
Const defaultExtension$ = "ts"
Dim Cache As CacheKeeper = Nothing
Using tmpPr As New PreProgress(Progress)
Try
@@ -59,10 +89,13 @@ Namespace API.Base
End If
End If
Dim p As SFileNumbers = SFileNumbers.Default(ConcatFile.Name)
Dim pNum As ANumbers = SFileNumbers.NumberProviderDefault
p.NumberProvider = pNum
DirectCast(p.NumberProvider, ANumbers).GroupSize = {URLs.Count.ToString.Length, 3}.Max
ConcatFile = SFile.IndexReindex(ConcatFile,,, p, EDP.ReturnValue)
Dim i%
Dim dFile As SFile = cache2.RootDirectory
dFile.Extension = "ts"
dFile.Extension = defaultExtension
Using w As New DownloadObjects.WebClient2(Responser)
For i = 0 To URLs.Count - 1
If progressExists Then
@@ -73,12 +106,14 @@ Namespace API.Base
End If
End If
Token.ThrowIfCancellationRequested()
dFile.Name = $"ConPart_{i}"
w.DownloadFile(URLs(i), dFile)
dFile.Name = $"{TempFilePrefix}{i.NumToString(pNum)}"
dFile.Extension = URLs(i).Extension.IfNullOrEmpty(defaultExtension)
w.DownloadFile(URLs(i).URL, dFile)
cache2.AddFile(dFile, True)
Next
End Using
DestinationFile = FFMPEG.ConcatenateFiles(cache2, Settings.FfmpegFile.File, ConcatFile, Settings.CMDEncoding, p, EDP.ThrowException)
If Not OnlyDownload Then _
DestinationFile = FFMPEG.ConcatenateFiles(cache2, Settings.FfmpegFile.File, ConcatFile, Settings.CMDEncoding, p, EDP.ThrowException)
Return DestinationFile
End If
End If

View File

@@ -47,6 +47,7 @@ Namespace API.Base
Progress.InformationTemporary = $"{HOST.Name} ({c - s}/{c}) Images: {_TotalImages}; Videos: {_TotalVideos}"
End If
End If
If _FeedDataExists Then Downloader.Files.Sort()
End Sub
Private Overloads Sub Download(ByVal Host As SettingsHost, ByVal Number As Integer, ByVal Count As Integer,
ByVal Token As CancellationToken, ByVal Multiple As Boolean)

View File

@@ -14,5 +14,6 @@ Namespace API.Xhamster
Friend ReadOnly HtmlScript As RParams = RParams.DMS("\<script id='initials-script'\>window.initials=(\{.+?\});\</script\>", 1, EDP.ReturnValue,
CType(Function(Input$) Input.StringTrim, Func(Of String, String)))
Friend ReadOnly FirstM3U8FileRegEx As RParams = RParams.DM("RESOLUTION=\d+x(\d+).*?[\r\n]+?([^#]*?\.m3u8.*)", 0, RegexReturn.List)
Friend ReadOnly SecondM3U8FileRegEx As RParams = RParams.DM("(#EXT-X-MAP.URI=""([^""]+((?<=\.)([^\?\.]{2,5})(?=(\?|\Z|"")))(.+|))""|#EXTINF[^\r\n]*[\r\n]+(([^\r\n]+((?<=\.)([^\?\.\r\n]{2,5})(?=(\?[^\r\n]+|[\r\n]+)))([^\r\n]+|))([\r\n]+|\Z)))", 0, RegexReturn.List)
End Module
End Namespace

View File

@@ -10,6 +10,7 @@ Imports System.Threading
Imports SCrawler.API.Base
Imports SCrawler.API.Base.M3U8Declarations
Imports PersonalUtilities.Forms.Toolbars
Imports PersonalUtilities.Tools
Imports PersonalUtilities.Tools.Web.Clients
Imports PersonalUtilities.Functions.RegularExpressions
Namespace API.Xhamster
@@ -40,18 +41,40 @@ Namespace API.Xhamster
Next
Return String.Empty
End Function
Private Shared Function ParseSecondM3U8(ByVal URL As String, ByVal Responser As Responser, ByVal Appender As String) As List(Of String)
Private Shared Function ParseSecondM3U8(ByVal URL As String, ByVal Responser As Responser, ByVal Appender As String) As List(Of M3U8URL)
Dim r$
Dim l As List(Of String)
Dim ll As List(Of M3U8URL) = Nothing
Dim u As M3U8URL
Dim rmsF As Func(Of RegexMatchStruct, M3U8URL) =
Function(ByVal rms As RegexMatchStruct) As M3U8URL
With rms
If .Arr(0).IsEmptyString Then
Return New M3U8URL(.Arr(3).IfNullOrEmpty(.Arr(4)), .Arr(5).IfNullOrEmpty(.Arr(6)))
Else
Return New M3U8URL(.Arr(0), .Arr(1).IfNullOrEmpty(.Arr(2)))
End If
End With
End Function
For i% = 0 To 1
Try
Responser.UseGZipStream = i
r = Responser.GetResponse(URL)
If Not r.IsEmptyString Then
l = RegexReplace(r, TsFilesRegEx)
If l.ListExists Then
For indx% = 0 To l.Count - 1 : l(indx) = M3U8Base.CreateUrl(Appender, l(indx)) : Next
Return l
If Not l.ListExists Then
With RegexFields(Of RegexMatchStruct)(r, {SecondM3U8FileRegEx}, {2, 4, 3, 8, 7, 10, 9}, EDP.ReturnValue)
If .ListExists Then ll = .Select(rmsF).ListWithRemove(Function(v) v.URL.IsEmptyString)
End With
End If
If Not ll.ListExists And l.ListExists Then ll = l.ListCast(Of M3U8URL)
If ll.ListExists Then
For indx% = 0 To ll.Count - 1
u = ll(indx)
u.URL = M3U8Base.CreateUrl(Appender, u.URL)
ll(indx) = u
Next
Return ll
End If
End If
Catch
@@ -59,14 +82,14 @@ Namespace API.Xhamster
Next
Return Nothing
End Function
Private Shared Function ObtainUrls(ByVal URL As String, ByVal Responser As Responser, ByVal UHD As Boolean) As List(Of String)
Private Shared Function ObtainUrls(ByVal URL As String, ByVal Responser As Responser, ByVal UHD As Boolean) As List(Of M3U8URL)
Try
Dim file$ = ParseFirstM3U8(URL, Responser, UHD)
If Not file.IsEmptyString Then
Responser.UseGZipStream = False
Dim appender$ = URL.Replace(URL.Split("/").LastOrDefault, String.Empty)
URL = M3U8Base.CreateUrl(appender, file)
Dim l As List(Of String) = ParseSecondM3U8(URL, Responser, appender)
Dim l As List(Of M3U8URL) = ParseSecondM3U8(URL, Responser, appender)
If l.ListExists Then Return l
End If
Return Nothing
@@ -75,8 +98,57 @@ Namespace API.Xhamster
End Try
End Function
Friend Shared Function Download(ByVal Media As UserMedia, ByVal Responser As Responser, ByVal UHD As Boolean,
ByVal Token As CancellationToken, ByVal Progress As MyProgress, ByVal UsePreProgress As Boolean) As SFile
Return M3U8Base.Download(ObtainUrls(Media.URL, Responser, UHD), Media.File, Responser, Token, Progress, UsePreProgress)
ByVal Token As CancellationToken, ByVal Progress As MyProgress, ByVal UsePreProgress As Boolean,
ByVal ReencodeVideos As Boolean) As SFile
'Return M3U8Base.Download(ObtainUrls(Media.URL, Responser, UHD), Media.File, Responser, Token, Progress, UsePreProgress)
Dim Cache As CacheKeeper = Nothing
Try
Dim urls As List(Of M3U8URL) = ObtainUrls(Media.URL, Responser, UHD)
If urls.ListExists Then
Cache = New CacheKeeper($"{Media.File.PathWithSeparator}_{M3U8Base.TempCacheFolderName}\") With {.DisposeSuspended = True}
Cache.CacheDeleteError = CacheDeletionError(Cache)
Dim isNewWay As Boolean = Not urls(0).Extension.IsEmptyString AndAlso urls(0).Extension = "mp4" AndAlso urls.Count > 1 AndAlso
urls.Exists(Function(u) Not u.Extension = urls(0).Extension)
Dim f As SFile = M3U8Base.Download(urls, Media.File, Responser, Token, Progress, UsePreProgress, Cache, isNewWay)
If isNewWay Then
f = Media.File
With DirectCast(Cache.CurrentInstance, CacheKeeper)
If .Count > 0 Then
Using batch As New BatchExecutor With {.Encoding = Settings.CMDEncoding}
batch.ChangeDirectory(.Self.RootDirectory)
Using bat As New TextSaver($"{ .RootDirectory.PathWithSeparator}Merge.bat")
Dim tmpFile As SFile
Dim tmpFileStr$
If ReencodeVideos Then
tmpFile = $"{ .Self.RootDirectory.PathWithSeparator}NewVideo.{urls(1).Extension}"
tmpFileStr = tmpFile.File
.AddFile(tmpFile)
Else
tmpFile = Media.File
tmpFileStr = $"""{tmpFile}"""
End If
bat.AppendLine($"copy /b { .First.File} + {M3U8Base.TempFilePrefix}*.{urls(1).Extension} {tmpFileStr}")
If ReencodeVideos Then bat.AppendLine($"""{Settings.FfmpegFile}"" -i ""{tmpFile}"" ""{Media.File}""")
bat.Save()
.AddFile(bat.File)
batch.Execute($"""{bat.File}""")
If f.Exists Then Return f
End Using
End Using
End If
End With
ElseIf f.Exists Then
Return f
End If
End If
Return Nothing
Finally
Cache.DisposeIfReady(False)
End Try
End Function
End Class
End Namespace

View File

@@ -28,7 +28,11 @@ Namespace API.Xhamster
End Get
End Property
<PropertyOption(ControlText:="Download UHD", ControlToolTip:="Download UHD (4K) content"), PXML, PClonable>
Friend Property DownloadUHD As PropertyValue
Friend ReadOnly Property DownloadUHD As PropertyValue
<PropertyOption(ControlText:="Re-encode downloaded videos if necessary",
ControlToolTip:="If enabled and the video is downloaded in a non-native format, the video will be re-encoded." & vbCr &
"Attention! Enabling this setting results in maximum CPU usage."), PXML, PClonable>
Friend ReadOnly Property ReencodeVideos As PropertyValue
#End Region
#Region "Initializer"
Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
@@ -38,6 +42,7 @@ Namespace API.Xhamster
SiteDomains = New PropertyValue(Domains.DomainsDefault, GetType(String))
Domains.DestinationProp = SiteDomains
DownloadUHD = New PropertyValue(False)
ReencodeVideos = New PropertyValue(False)
_SubscriptionsAllowed = True
UrlPatternUser = "https://xhamster.com/{0}/{1}"

View File

@@ -537,7 +537,7 @@ Namespace API.Xhamster
Private Overloads Function GetM3U8(ByRef m As UserMedia, ByVal j As EContainer) As Boolean
Dim node As EContainer = j({"xplayerSettings", "sources", "hls"})
If node.ListExists Then
Dim url$ = node.GetNode({New NodeParams("url", True, True, True, True, 2)})
Dim url$ = node.GetNode({New NodeParams("url", True, True, True, True, 2)}).XmlIfNothingValue
If Not url.IsEmptyString Then m.URL = url : m.Type = UTypes.m3u8 : Return True
End If
Return False
@@ -555,7 +555,7 @@ Namespace API.Xhamster
End Sub
Protected Overrides Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Token As CancellationToken) As SFile
Media.File = DestinationFile
Return M3U8.Download(Media, Responser, MySettings.DownloadUHD.Value, Token, Progress, Not IsSingleObjectDownload)
Return M3U8.Download(Media, Responser, MySettings.DownloadUHD.Value, Token, Progress, Not IsSingleObjectDownload, MySettings.ReencodeVideos.Value)
End Function
#End Region
#Region "Create media"

View File

@@ -175,7 +175,7 @@ Namespace API.YouTube
maxDate = Nothing
LastDownloadDatePlaylist = nDate(LastDownloadDatePlaylist)
url = $"https://{IIf(IsMusic, "music", "www")}.youtube.com/playlist?list={ID}"
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr, True, False,, LastDownloadDatePlaylist)
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr,, LastDownloadDatePlaylist,, True)
applySpecFolder.Invoke(String.Empty, False)
If fillList.Invoke(LastDownloadDatePlaylist) Then LastDownloadDatePlaylist = If(maxDate, Now)
ElseIf YTMediaType = YouTubeMediaType.Channel Then
@@ -183,7 +183,7 @@ Namespace API.YouTube
maxDate = Nothing
LastDownloadDateVideos = nDate(LastDownloadDateVideos)
url = $"https://{IIf(IsMusic, "music", "www")}.youtube.com/{IIf(IsMusic Or IsChannelUser, $"{YouTubeFunctions.UserChannelOption}/", "@")}{ID}"
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr, True, False,, LastDownloadDateVideos)
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr,, LastDownloadDateVideos,, True)
applySpecFolder.Invoke(IIf(IsMusic, String.Empty, "Videos"), False)
If fillList.Invoke(LastDownloadDateVideos) Then LastDownloadDateVideos = If(maxDate, Now)
End If
@@ -191,7 +191,7 @@ Namespace API.YouTube
maxDate = Nothing
LastDownloadDateShorts = nDate(LastDownloadDateShorts)
url = $"https://www.youtube.com/{IIf(IsChannelUser, $"{YouTubeFunctions.UserChannelOption}/", "@")}{ID}/shorts"
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr, True, False,, LastDownloadDateShorts)
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr,, LastDownloadDateShorts,, True)
applySpecFolder.Invoke("Shorts", False)
If fillList.Invoke(LastDownloadDateShorts) Then LastDownloadDateShorts = If(maxDate, Now)
End If
@@ -199,7 +199,7 @@ Namespace API.YouTube
maxDate = Nothing
LastDownloadDatePlaylist = nDate(LastDownloadDatePlaylist)
url = $"https://www.youtube.com/{IIf(IsChannelUser, $"{YouTubeFunctions.UserChannelOption}/", "@")}{ID}/playlists"
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr, True, False,, LastDownloadDatePlaylist)
container = YouTubeFunctions.Parse(url, YTUseCookies, Token, pr,, LastDownloadDatePlaylist,, True)
applySpecFolder.Invoke("Playlists", True)
If fillList.Invoke(LastDownloadDatePlaylist) Then LastDownloadDatePlaylist = If(maxDate, Now)
End If

View File

@@ -25,8 +25,6 @@ Namespace DownloadObjects.STDownloader
AppMode = False
Icon = My.Resources.ArrowDownIcon_Blue_24
BTT_ADD_PLS_ARR.Text = $"YouTube: {BTT_ADD_PLS_ARR.Text}"
BTT_ADD_NO_SHORTS.Text = $"YouTube: {BTT_ADD_NO_SHORTS.Text}"
BTT_ADD_SHORTS_ONLY.Text = $"YouTube: {BTT_ADD_SHORTS_ONLY.Text}"
BTT_ADD_URLS_ARR = New ToolStripMenuItemKeyClick("Add an array of URLs", PersonalUtilities.My.Resources.PlusPic_Green_24) With {.Tag = UrlsArrTag}
BTT_ADD_URLS_EXTERNAL = New ToolStripMenuItemKeyClick With {.Tag = TAG_EXTERNAL}
MENU_ADD.DropDownItems.Insert(1, BTT_ADD_URLS_ARR)

View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("2023.12.10.0")>
<Assembly: AssemblyFileVersion("2023.12.10.0")>
<Assembly: AssemblyVersion("2023.12.13.0")>
<Assembly: AssemblyFileVersion("2023.12.13.0")>
<Assembly: NeutralResourcesLanguage("en")>

View File

@@ -70,6 +70,9 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
Public Shared Widening Operator CType(ByVal f As ProgramFile) As SFile
Return f.File
End Operator
Public Shared Narrowing Operator CType(ByVal f As ProgramFile) As String
Return f.ToString
End Operator
Public Overrides Function ToString() As String
Return File.ToString
End Function