geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

2024.10.24.0

Browse Source 
YT
YouTubeSettings: add 'DefaultVideoAllowWebm' and 'DefaultAudioEmbedThumbnail_Cover' settings
YouTubeMediaContainerBase: change cover selection for music download; fix adding incorrect playlist lines; allow 'webm' formats is there are no 'mp4' formats via http protocol

SCrawler
DeclaredNames: add new names
UserDataBase: add '_ForceSaveUserInfoOnException' field  and 'UpdateUserInformation_Ex' function to update user info on exception; clear '_MD5List' when clearing data and/or history
API.Instagram: add manual 'UserName' changing; mark user as non-existent if user ID cannot be obtained
API.Twitter: add manual 'UserName' changing
API.Mastodon: bypass inherited property
API.Reddit: fix incorrect UNIX date parsing
DownloadFeedForm: add exception handling to the 'RefillAfterDelete' function
MainFrame: add 'MENU_INFO_USER_SEARCH' to the 'Info' menu
SettingsHostCollection: fix a bug when changing data paths
This commit is contained in:
Andy
2024-10-24 19:18:29 +03:00
parent 00a06d3e9a
commit aedcebc781
25 changed files with 172 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
SCrawler.YouTube/Base/YouTubeSettings.vb
View File

@@ -311,6 +311,9 @@ Namespace API.YouTube.Base
<Browsable(True), GridVisible, XMLVN({"DefaultsVideo"}, 1080), Category("Defaults Video"), DisplayName("Default definition"),
Description("The default maximum video resolution. -1 for max definition")>
Public ReadOnly Property DefaultVideoDefinition As XMLValue(Of Integer)
<Browsable(True), GridVisible, XMLVN({"DefaultsVideo"}, True), Category("Defaults Video"), DisplayName("Allow webm formats"),
Description("Allow webm formats over http if mp4 formats are not available. Default: true.")>
Public ReadOnly Property DefaultVideoAllowWebm As XMLValue(Of Boolean)
<Browsable(True), GridVisible, XMLVN({"DefaultsVideo"}), Category("Defaults Video"), DisplayName("Convert non-AVC codecs to AVC"),
Description("Convert non-AVC codecs (eg 'VP9') to AVC. Not recommended due to high CPU usage!")>
Public ReadOnly Property DefaultVideoConvertNonAVC As XMLValue(Of Boolean)
@@ -416,6 +419,9 @@ Namespace API.YouTube.Base
<Browsable(True), GridVisible, XMLVN({"DefaultsAudio"}, True), Category("Defaults Audio"), DisplayName("Embed thumbnail"),
Description("Embed thumbnail in the audio as cover art. Default: true.")>
Public ReadOnly Property DefaultAudioEmbedThumbnail As XMLValue(Of Boolean)
<Browsable(True), GridVisible, XMLVN({"DefaultsAudio"}, True), Category("Defaults Audio"), DisplayName("Embed thumbnail (cover)"),
Description("Try embedding the playlist cover (if it exists) as cover art. Default: true.")>
Public ReadOnly Property DefaultAudioEmbedThumbnail_Cover As XMLValue(Of Boolean)
<Browsable(True), GridVisible, XMLVN({"DefaultsAudio"}, True), Category("Defaults Audio"), DisplayName("Embed thumbnail (extracted files)"),
Description("Embed thumbnail in the extracted (additional file ('mp3' only)) audio as cover art. Default: true.")>
Public ReadOnly Property DefaultAudioEmbedThumbnail_ExtractedFiles As XMLValue(Of Boolean)

1
SCrawler.YouTube/Declarations.vb
View File

@@ -59,6 +59,7 @@ Namespace API.YouTube
Public ReadOnly TrueUrlRegEx As RParams = RParams.DM(Base.YouTubeFunctions.TrueUrlPattern, 0, EDP.ReturnValue)
Friend ReadOnly MusicUrlApply As RParams = RParams.DMS("https://([w\.]*)youtube.com.+", 1, RegexReturn.Replace, EDP.ReturnValue,
CType(Function(input$) "music.", Func(Of String, String)), String.Empty)
Friend ReadOnly M3U8ExcludedSymbols As String() = {".", ",", ":", "/", "\", "(", ")", "[", "]"}
<Extension> Friend Function ToMusicUrl(ByVal URL As String, ByVal IsMusic As Boolean) As String
Try : Return If(IsMusic And Not URL.IsEmptyString, CStr(RegexReplace(URL, MusicUrlApply)).IfNullOrEmpty(URL), URL) : Catch : Return URL : End Try
End Function

4
SCrawler.YouTube/My Project/AssemblyInfo.vb
View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("2024.8.10.0")>
<Assembly: AssemblyFileVersion("2024.8.10.0")>
<Assembly: AssemblyVersion("2024.10.24.0")>
<Assembly: AssemblyFileVersion("2024.10.24.0")>
<Assembly: NeutralResourcesLanguage("en")>

43
SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb
View File

@@ -175,7 +175,9 @@ Namespace API.YouTube.Objects
Protected _ThumbnailUrl As String = String.Empty
<XMLEC> Public Overridable Property ThumbnailUrl As String Implements IDownloadableMedia.ThumbnailUrl
Get
If _ThumbnailUrl.IsEmptyString And Thumbnails.Count > 0 Then
If Not CoverURL.IsEmptyString Then
Return CoverURL
ElseIf _ThumbnailUrl.IsEmptyString And Thumbnails.Count > 0 Then
Return Thumbnails.FirstOrDefault.URL
Else
Return _ThumbnailUrl
@@ -904,7 +906,8 @@ Namespace API.YouTube.Objects
Const m3u8DataRow$ = "#EXTINF:{0},{1}" & vbCrLf & "{2}"
With Element
Dim f As SFile = __file.IfNullOrEmpty(.File)
Dim __f$ = SymbolsConverter.ASCII.EncodeSymbolsOnly(If(Mode = M3U8CreationMode.Absolute, f.ToString, f.File))
Dim fStr$ = f.ToString.StringReplaceSymbols({"\"}, "/", EDP.ReturnValue)
Dim __f$ = SymbolsConverter.ASCII.Extended.EncodeSymbolsOnly(If(Mode = M3U8CreationMode.Absolute, fStr, f.File), M3U8ExcludedSymbols)
If Mode = M3U8CreationMode.Absolute Then __f = $"file:///{__f}"
Dim fName$ = .Title.IfNullOrEmpty(f.Name)
If MyYouTubeSettings.MusicPlaylistCreate_M3U8_AppendNumber And .PlaylistIndex > 0 Then fName = $"{ .PlaylistIndex}. {fName}"
@@ -1022,12 +1025,19 @@ Namespace API.YouTube.Objects
End If
Dim cDown As Boolean = False
Dim fCover As SFile = Nothing
Dim cUrl$ = String.Empty
For Each elem In Elements
With DirectCast(elem, YouTubeMediaContainerBase)
If Not .CoverDownloaded Then .CoverDownloaded = cDown
'If Not .CoverDownloaded Then .CoverDownloaded = cDown
.CoverDownloaded = cDown
.CoverFile = fCover
.CoverURL = cUrl
AddHandler .FileDownloadStarted, fDown
.Download(UseCookies, Token)
cDown = .CoverDownloaded
fCover = .CoverFile
cUrl = .CoverURL
RemoveHandler .FileDownloadStarted, fDown
End With
If Token.IsCancellationRequested Or disposedValue Then Exit For
@@ -1054,6 +1064,8 @@ Namespace API.YouTube.Objects
End Try
End Sub
Protected CoverDownloaded As Boolean = False
Protected CoverFile As SFile = Nothing
Protected CoverURL As String = String.Empty
Private Sub DownloadPlaylistCover(ByVal PlsId As String, ByVal f As SFile, ByVal UseCookies As Boolean)
Try
Dim url$ = $"https://{IIf(IsMusic, "music", "www")}.youtube.com/playlist?list={PlsId}"
@@ -1089,7 +1101,8 @@ Namespace API.YouTube.Objects
url = LinkFormatterSecure(u)
f.Name = "cover"
f.Extension = "jpg"
If resp.DownloadFile(url, f, EDP.ReturnValue) And f.Exists Then CoverDownloaded = True : AddFile(f)
If resp.DownloadFile(url, f, EDP.ReturnValue) And f.Exists Then _
CoverFile = f : CoverURL = url : CoverDownloaded = True : AddFile(f)
End If
End If
End Using
@@ -1270,10 +1283,10 @@ Namespace API.YouTube.Objects
End Sub
Dim embedThumbTo As Action(Of SFile) =
Sub(ByVal dFile As SFile)
If dFile.Exists And ThumbnailFile.Exists Then
If dFile.Exists And CoverFile.IfNullOrEmpty(ThumbnailFile).Exists Then
Dim dFileNew As SFile = dFile
dFileNew.Name &= "_NEW"
.Execute($"ffmpeg -i ""{dFile}"" -i ""{ThumbnailFile}"" -map 0:0 -map 1:0 -c copy -id3v2_version 3 -metadata:s:v title=""Cover"" -metadata:s:v comment=""Cover"" ""{dFileNew}""")
.Execute($"ffmpeg -i ""{dFile}"" -i ""{CoverFile.IfNullOrEmpty(ThumbnailFile)}"" -map 0:0 -map 1:0 -c copy -id3v2_version 3 -metadata:s:v title=""Cover"" -metadata:s:v comment=""Cover"" ""{dFileNew}""")
If dFileNew.Exists AndAlso dFile.Delete(,, EDP.ReturnValue) Then SFile.Rename(dFileNew, dFile)
End If
End Sub
@@ -1353,6 +1366,10 @@ Namespace API.YouTube.Objects
End If
End If
'mp3
If IsMusic And ObjectType = YouTubeMediaType.Single And File.Extension = mp3 And
Not mp3ThumbEmbedded And CoverFile.Exists And MyYouTubeSettings.DefaultAudioEmbedThumbnail_Cover Then embedThumbTo.Invoke(File)
'Update video
ThrowAny(Token)
If SelectedVideoIndex >= 0 AndAlso tempFilesList.Count > 0 AndAlso tempFilesList.Exists(Function(tf) tf.ToReplace) Then
@@ -1725,6 +1742,7 @@ Namespace API.YouTube.Objects
Dim obj As MediaObject
Dim nValue#
Dim sValue$
Dim allowWebm As Boolean = MyYouTubeSettings.DefaultVideoAllowWebm
Dim validCodecValue As Func(Of String, Boolean) = Function(codec) Not codec.IsEmptyString AndAlso Not codec = "none"
For Each ee In e({"formats"})
@@ -1775,12 +1793,13 @@ Namespace API.YouTube.Objects
Dim d As MediaObject = Nothing
Dim expWebm As Predicate(Of MediaObject) = Function(mo) mo.Extension = webm
Dim expAVC As Predicate(Of MediaObject) = Function(mo) mo.Codec.IfNullOrEmpty("/").ToLower.StartsWith(avc)
Dim comp As Func(Of MediaObject, Predicate(Of MediaObject), Boolean, Boolean) =
Function(mo, exp, isTrue) mo.Type = t And exp.Invoke(mo) = isTrue And mo.Width = d.Width
Dim CountWebm As Func(Of MediaObject, Boolean) = Function(mo) comp.Invoke(mo, expWebm, False)
Dim RemoveWebm As Predicate(Of MediaObject) = Function(mo) comp.Invoke(mo, expWebm, True)
Dim CountAVC As Func(Of MediaObject, Boolean) = Function(mo) comp.Invoke(mo, expAVC, True)
Dim RemoveAVC As Predicate(Of MediaObject) = Function(mo) comp.Invoke(mo, expAVC, False)
Dim comp As Func(Of MediaObject, Predicate(Of MediaObject), Boolean, Boolean, Boolean) =
Function(mo, exp, isTrue, checkHttp) mo.Type = t And exp.Invoke(mo) = isTrue And mo.Width = d.Width And
(Not checkHttp OrElse mo.ProtocolType = Protocols.https)
Dim CountWebm As Func(Of MediaObject, Boolean) = Function(mo) comp.Invoke(mo, expWebm, False, allowWebm)
Dim RemoveWebm As Predicate(Of MediaObject) = Function(mo) comp.Invoke(mo, expWebm, True, allowWebm)
Dim CountAVC As Func(Of MediaObject, Boolean) = Function(mo) comp.Invoke(mo, expAVC, True, False)
Dim RemoveAVC As Predicate(Of MediaObject) = Function(mo) comp.Invoke(mo, expAVC, False, False)
For Each d In data
If MediaObjects.Count = 0 Then Exit For
If MediaObjects.LongCount(CountWebm) > 0 Then MediaObjects.RemoveAll(RemoveWebm)