From 59dacf67bd166952cdefcae55c2aaf1920cdc752 Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:59:40 +0300 Subject: [PATCH] 2026.10.26.0 YT Exclude AI from results and correct handling SC API.UserDataBase: force put text into the 'text' folder API.Instagram: fix saved posts API.OnlyFans: text posts are not downloading API.Xhamster: correct processing progress --- Changelog.md | 17 +++- SCrawler.YouTube/Base/Structures.vb | 45 ++++++++++ SCrawler.YouTube/Base/YouTubeSettings.vb | 9 ++ .../Controls/VideoOption.Designer.vb | 50 +++++------ SCrawler.YouTube/Controls/VideoOption.resx | 3 - SCrawler.YouTube/Controls/VideoOption.vb | 82 ++++++++++++++++--- SCrawler.YouTube/Controls/VideoOptionsForm.vb | 50 +++++++++-- SCrawler.YouTube/Declarations.vb | 6 +- SCrawler.YouTube/My Project/AssemblyInfo.vb | 6 +- .../Objects/YouTubeMediaContainerBase.vb | 36 +++++--- .../My Project/AssemblyInfo.vb | 6 +- SCrawler/API/Base/UserDataBase.vb | 8 +- SCrawler/API/Base/YTDLP.vb | 4 +- SCrawler/API/Instagram/UserData.vb | 4 +- SCrawler/API/OnlyFans/UserData.vb | 8 +- SCrawler/API/Xhamster/UserData.vb | 2 +- SCrawler/My Project/AssemblyInfo.vb | 6 +- 17 files changed, 266 insertions(+), 76 deletions(-) diff --git a/Changelog.md b/Changelog.md index dee7007..cbcaa10 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,13 +2,26 @@ - [ffmpeg](https://github.com/AAndyProgram/SCrawler/wiki/Settings#ffmpeg) - x64 version - [release](https://github.com/GyanD/codexffmpeg/releases/tag/5.1.2); [zip](https://github.com/GyanD/codexffmpeg/releases/download/5.1.2/ffmpeg-5.1.2-full_build.zip); **version `5.1.2-full_build-www.gyan.dev`** - x86 version - [release](https://github.com/yt-dlp/FFmpeg-Builds/releases/tag/autobuild-2022-11-30-12-57); [zip](https://github.com/yt-dlp/FFmpeg-Builds/releases/download/autobuild-2022-11-30-12-57/ffmpeg-N-109274-gd7a5f068c2-win32-gpl.zip); **version `N-109457-geeb280f351-20221226`** -- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.32.13** -- [YT-DLP](https://github.com/AAndyProgram/SCrawler/wiki/Settings#yt-dlp) - **2026.08.19** +- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.32.18** +- [YT-DLP](https://github.com/AAndyProgram/SCrawler/wiki/Settings#yt-dlp) - **2026.10.21** - [Deno](https://github.com/AAndyProgram/SCrawler/wiki/Settings#deno) - latest *(`2.0.0` or higher)* - [OF-Scraper](https://github.com/AAndyProgram/SCrawler/wiki/Settings#of-scraper) - **3.12.9** ([release](https://github.com/datawhores/OF-Scraper/releases/tag/3.12.9)) # 2026 +## 2026.10.26.0 + +*2026-10-26* + +- Added + - **YouTube downloader: correct processing of AI results** + - Minor improvements +- Updated + - gallery-dl up to version **1.32.18** + - yt-dlp up to version **2026.10.21** +- Fixed + - Minor bugs + ## 2026.9.21.0 *2026-09-21* diff --git a/SCrawler.YouTube/Base/Structures.vb b/SCrawler.YouTube/Base/Structures.vb index 3fef50a..070918f 100644 --- a/SCrawler.YouTube/Base/Structures.vb +++ b/SCrawler.YouTube/Base/Structures.vb @@ -113,6 +113,51 @@ Namespace API.YouTube.Base Return Protocols.Undefined End Get End Property + Public Language As String + Public ReadOnly Property LanguageCaption(Optional ByVal IncludeNote As Boolean = False) As String + Get + Dim c$ = String.Empty + If Not Language.IsEmptyString Then c = $"[{Language}]" + If Not FormatNote.IsEmptyString And IncludeNote Then c.StringAppend($"({FormatNoteClean.IfNullOrEmpty(FormatNote)})", " ") + Return c + End Get + End Property + Public FormatNote As String + Public FormatNoteClean As String + Public ReadOnly Property IsAI As Boolean + Get + Return Not FormatNote.IsEmptyString AndAlso FormatNote.ToLower.Contains("dubbed-auto") + End Get + End Property + Public ReadOnly Property IsOriginalLanguage As Boolean + Get + Return Not FormatNote.IsEmptyString AndAlso FormatNote.ToLower.Contains("original") + End Get + End Property + Public ReadOnly Property IsDefaultLanguage As Boolean + Get + Return Not FormatNote.IsEmptyString AndAlso FormatNote.ToLower.Contains("default") + End Get + End Property + Public ReadOnly Property IsDefaultOrOriginal As Boolean + Get + Return IsDefaultLanguage Or IsOriginalLanguage + End Get + End Property + Public ReadOnly Property FormatQuality As String + Get + If Not FormatNote.IsEmptyString Then + If FormatNote.Contains("high") Then + Return "high" + ElseIf FormatNote.Contains("medium") Then + Return "medium" + ElseIf FormatNote.Contains("low") Then + Return "low" + End If + End If + Return String.Empty + End Get + End Property Public URL As String Public Property Index As Integer Implements IIndexable.Index Private Function SetIndex(ByVal Obj As Object, ByVal Index As Integer) As Object Implements IIndexable.SetIndex diff --git a/SCrawler.YouTube/Base/YouTubeSettings.vb b/SCrawler.YouTube/Base/YouTubeSettings.vb index 5096951..c4f6109 100644 --- a/SCrawler.YouTube/Base/YouTubeSettings.vb +++ b/SCrawler.YouTube/Base/YouTubeSettings.vb @@ -413,6 +413,15 @@ Namespace API.YouTube.Base Private Function AvailableAudioFormats_Impl() As String() Return AvailableAudioFormats End Function + + Public ReadOnly Property DefaultAudioLanguage_ExcludeAI As XMLValue(Of Boolean) + + Public ReadOnly Property DefaultAudioLanguage_ExcludeNonDefault As XMLValue(Of Boolean) + + Public ReadOnly Property DefaultAudioLanguageUseDefault As XMLValue(Of Boolean) diff --git a/SCrawler.YouTube/Controls/VideoOption.Designer.vb b/SCrawler.YouTube/Controls/VideoOption.Designer.vb index 15861b7..7db0b0d 100644 --- a/SCrawler.YouTube/Controls/VideoOption.Designer.vb +++ b/SCrawler.YouTube/Controls/VideoOption.Designer.vb @@ -22,37 +22,36 @@ Namespace API.YouTube.Controls Private components As System.ComponentModel.IContainer Private Sub InitializeComponent() - Dim TP_MAIN As System.Windows.Forms.TableLayoutPanel + Me.TP_MAIN = New System.Windows.Forms.TableLayoutPanel() Me.OPT_CHECKED = New System.Windows.Forms.RadioButton() Me.LBL_DEFINITION_INFO = New System.Windows.Forms.Label() Me.LBL_DEFINITION = New System.Windows.Forms.Label() Me.LBL_CODECS = New System.Windows.Forms.Label() Me.LBL_SIZE = New System.Windows.Forms.Label() - TP_MAIN = New System.Windows.Forms.TableLayoutPanel() - TP_MAIN.SuspendLayout() + Me.TP_MAIN.SuspendLayout() Me.SuspendLayout() ' 'TP_MAIN ' - TP_MAIN.ColumnCount = 5 - TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 36.0!)) - TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 110.0!)) - TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 58.0!)) - TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) - TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 71.0!)) - TP_MAIN.Controls.Add(Me.OPT_CHECKED, 0, 0) - TP_MAIN.Controls.Add(Me.LBL_DEFINITION_INFO, 1, 0) - TP_MAIN.Controls.Add(Me.LBL_DEFINITION, 2, 0) - TP_MAIN.Controls.Add(Me.LBL_CODECS, 3, 0) - TP_MAIN.Controls.Add(Me.LBL_SIZE, 4, 0) - TP_MAIN.Dock = System.Windows.Forms.DockStyle.Fill - TP_MAIN.Location = New System.Drawing.Point(0, 0) - TP_MAIN.Margin = New System.Windows.Forms.Padding(0) - TP_MAIN.Name = "TP_MAIN" - TP_MAIN.RowCount = 1 - TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) - TP_MAIN.Size = New System.Drawing.Size(459, 30) - TP_MAIN.TabIndex = 0 + Me.TP_MAIN.ColumnCount = 5 + Me.TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 36.0!)) + Me.TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 110.0!)) + Me.TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 58.0!)) + Me.TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) + Me.TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 71.0!)) + Me.TP_MAIN.Controls.Add(Me.OPT_CHECKED, 0, 0) + Me.TP_MAIN.Controls.Add(Me.LBL_DEFINITION_INFO, 1, 0) + Me.TP_MAIN.Controls.Add(Me.LBL_DEFINITION, 2, 0) + Me.TP_MAIN.Controls.Add(Me.LBL_CODECS, 3, 0) + Me.TP_MAIN.Controls.Add(Me.LBL_SIZE, 4, 0) + Me.TP_MAIN.Dock = System.Windows.Forms.DockStyle.Fill + Me.TP_MAIN.Location = New System.Drawing.Point(0, 0) + Me.TP_MAIN.Margin = New System.Windows.Forms.Padding(0) + Me.TP_MAIN.Name = "TP_MAIN" + Me.TP_MAIN.RowCount = 1 + Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) + Me.TP_MAIN.Size = New System.Drawing.Size(459, 30) + Me.TP_MAIN.TabIndex = 0 ' 'OPT_CHECKED ' @@ -115,11 +114,11 @@ Namespace API.YouTube.Controls ' Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!) Me.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font - Me.Controls.Add(TP_MAIN) + Me.Controls.Add(Me.TP_MAIN) Me.Name = "VideoOption" Me.Size = New System.Drawing.Size(459, 30) - TP_MAIN.ResumeLayout(False) - TP_MAIN.PerformLayout() + Me.TP_MAIN.ResumeLayout(False) + Me.TP_MAIN.PerformLayout() Me.ResumeLayout(False) End Sub @@ -128,5 +127,6 @@ Namespace API.YouTube.Controls Private WithEvents LBL_DEFINITION As Label Private WithEvents LBL_CODECS As Label Private WithEvents LBL_SIZE As Label + Private WithEvents TP_MAIN As TableLayoutPanel End Class End Namespace \ No newline at end of file diff --git a/SCrawler.YouTube/Controls/VideoOption.resx b/SCrawler.YouTube/Controls/VideoOption.resx index e5f5c30..1af7de1 100644 --- a/SCrawler.YouTube/Controls/VideoOption.resx +++ b/SCrawler.YouTube/Controls/VideoOption.resx @@ -117,7 +117,4 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - False - \ No newline at end of file diff --git a/SCrawler.YouTube/Controls/VideoOption.vb b/SCrawler.YouTube/Controls/VideoOption.vb index 7066658..5eacf2c 100644 --- a/SCrawler.YouTube/Controls/VideoOption.vb +++ b/SCrawler.YouTube/Controls/VideoOption.vb @@ -8,10 +8,14 @@ ' but WITHOUT ANY WARRANTY Imports System.ComponentModel Imports SCrawler.API.YouTube.Base +Imports SCrawler.API.YouTube.Objects +Imports UMTypes = SCrawler.Plugin.UserMediaTypes Namespace API.YouTube.Controls Friend Class VideoOption : Implements ISupportInitialize Friend Event CheckedChanged As EventHandler + Private Const DRC As String = YouTubeMediaContainerBase.DRC + Private Const StringDelimiter As String = " " & ChrW(183) & " " Friend Property MyMedia As MediaObject Friend Property Checked As Boolean Get @@ -21,16 +25,43 @@ Namespace API.YouTube.Controls OPT_CHECKED.Checked = _Checked End Set End Property + Friend ReadOnly Property ContentType As UMTypes + Get + If MyMedia.Type = UMTypes.Audio Then + Return UMTypes.Audio + ElseIf MyMedia.Type = UMTypes.Video Or MyMedia.Type = YouTubeMediaContainerBase.TypeAudioVideo Then + Return UMTypes.Video + Else + Return UMTypes.Undefined + End If + End Get + End Property + Private _CodecLength As Integer = 0 + Friend ReadOnly Property CodecLengthBase As Integer + Get + With TP_MAIN.ColumnStyles + Return .Item(0).Width + .Item(1).Width + .Item(2).Width + .Item(4).Width + End With + End Get + End Property + Friend ReadOnly Property CodecLength As Integer + Get + Return Math.Max(LBL_CODECS.PreferredWidth, _CodecLength) + End Get + End Property + Friend Sub SetAudioChangingHandler(ByVal obj As VideoOptionsForm) + AddHandler obj.AudioIndexChanged, AddressOf AudioChanged + End Sub Friend Sub New() InitializeComponent() End Sub Friend Sub New(ByVal m As MediaObject, Optional ByVal SelectedAudio As MediaObject = Nothing) Me.New - Const d$ = " " & ChrW(183) & " " - Const DRC$ = Objects.YouTubeMediaContainerBase.DRC MyMedia = m If m.Type = Plugin.UserMediaTypes.Audio Then - If m.Bitrate >= 320 Then + If Not m.FormatQuality.IsEmptyString Then + LBL_DEFINITION_INFO.Text = $"{m.FormatQuality.ToUpperFirstChar} Quality" + ElseIf m.Bitrate >= 320 Then LBL_DEFINITION_INFO.Text = "High Quality" ElseIf m.Bitrate >= 190 Then LBL_DEFINITION_INFO.Text = "Medium Quality" @@ -38,8 +69,8 @@ Namespace API.YouTube.Controls LBL_DEFINITION_INFO.Text = "Low Quality" End If LBL_DEFINITION.Text = $"{m.Bitrate}k" - LBL_CODECS.Text = $"{m.Extension} {d} {m.Codec} {d} {m.Bitrate}k" - If Not m.ID.IsEmptyString AndAlso m.ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $" {d} DRC" + 'LBL_CODECS.Text = $"{m.Extension} {d} {m.Codec} {d} {m.Bitrate}k{If(m.LanguageCaption.IsEmptyString, String.Empty, $" {d} {m.LanguageCaption}")}" + 'If Not m.ID.IsEmptyString AndAlso m.ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $" {d} DRC" Else If m.Height >= 1440 Then LBL_DEFINITION_INFO.Text = "Ultra High Definition" @@ -53,11 +84,13 @@ Namespace API.YouTube.Controls LBL_DEFINITION_INFO.Text = "Low Definition" End If LBL_DEFINITION.Text = $"{m.Height}p" - LBL_CODECS.Text = $"{m.Extension.StringToUpper}{d}{m.Codec.StringToUpper}{d}{m.FPS}fps{d}{m.Bitrate}k" - If Not m.Protocol.IsEmptyString Then LBL_CODECS.Text &= $" ({m.Protocol})" - If Not m.ID.IsEmptyString AndAlso m.ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $"{d}DRC" - If Not SelectedAudio.ID.IsEmptyString Then LBL_CODECS.Text &= $" / {SelectedAudio.Extension}{d}{SelectedAudio.Codec}{d}{SelectedAudio.Bitrate}k" - If Not SelectedAudio.ID.IsEmptyString AndAlso SelectedAudio.ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $"{d}DRC" + 'LBL_CODECS.Text = $"{m.Extension.StringToUpper}{d}{m.Codec.StringToUpper}{d}{m.FPS}fps{d}{m.Bitrate}k" + 'If Not m.Protocol.IsEmptyString Then LBL_CODECS.Text &= $" ({m.Protocol})" + 'If Not m.ID.IsEmptyString AndAlso m.ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $"{d}DRC" + 'With SelectedAudio + ' If Not .ID.IsEmptyString Then LBL_CODECS.Text &= $" / {If(.LanguageCaption.IsEmptyString, String.Empty, $"{ .LanguageCaption} {d}")}{ .Bitrate}k ({ .Extension}, { .Codec})" + ' If Not .ID.IsEmptyString AndAlso .ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $"{d}DRC" + 'End With If MyYouTubeSettings.DefaultVideoHighlightFPS_H > 0 AndAlso m.FPS > MyYouTubeSettings.DefaultVideoHighlightFPS_H Then _ BackColor = MyColor.DeleteBack : ForeColor = MyColor.DeleteFore @@ -65,6 +98,8 @@ Namespace API.YouTube.Controls BackColor = MyColor.UpdateBack : ForeColor = MyColor.UpdateFore End If + GenerateInfoCaption(SelectedAudio) + Dim sv% = m.Size / 1024 If sv >= 1000 Then LBL_SIZE.Text = AConvert(Of String)(sv / 1024, VideoSizeProvider) @@ -74,9 +109,36 @@ Namespace API.YouTube.Controls LBL_SIZE.Text &= " MB" End If End Sub + Private Sub GenerateInfoCaption(ByVal SelectedAudio As MediaObject) + Const d$ = StringDelimiter + If ContentType = UMTypes.Audio Then + With MyMedia + LBL_CODECS.Text = $"{ .Extension} {d} { .Codec} {d} { .Bitrate}k{If(.LanguageCaption.IsEmptyString, String.Empty, $" {d} { .LanguageCaption(True)}")}" + If Not .ID.IsEmptyString AndAlso .ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $" {d} DRC" + End With + Else + With MyMedia + LBL_CODECS.Text = $"{ .Extension.StringToUpper}{d}{ .Codec.StringToUpper}{d}{ .FPS}fps{d}{ .Bitrate}k" + If Not .Protocol.IsEmptyString Then LBL_CODECS.Text &= $" ({ .Protocol})" + If Not .ID.IsEmptyString AndAlso .ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $"{d}DRC" + _CodecLength = LBL_CODECS.Text.MeasureTextDefault(LBL_CODECS.Font).Width + End With + With SelectedAudio + If Not .ID.IsEmptyString Then LBL_CODECS.Text &= $" / {If(.LanguageCaption.IsEmptyString, String.Empty, $"{ .LanguageCaption} {d}")}{ .Bitrate}k ({ .Extension}, { .Codec})" + If Not .ID.IsEmptyString AndAlso .ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $"{d}DRC" + End With + End If + End Sub Private Sub OPT_CHECKED_CheckedChanged(sender As Object, e As EventArgs) Handles OPT_CHECKED.CheckedChanged RaiseEvent CheckedChanged(Me, e) End Sub + Private Sub AudioChanged(ByVal Sender As Object, ByVal e As EventArgs) + If ContentType = UMTypes.Video AndAlso Not Sender Is Nothing Then + With DirectCast(Sender, VideoOption) + If .ContentType = UMTypes.Audio And .Checked Then GenerateInfoCaption(.MyMedia) + End With + End If + End Sub Private Sub Labels_Click(sender As Object, e As EventArgs) Handles LBL_DEFINITION_INFO.Click, LBL_DEFINITION.Click, LBL_CODECS.Click, LBL_SIZE.Click OPT_CHECKED.Checked = True End Sub diff --git a/SCrawler.YouTube/Controls/VideoOptionsForm.vb b/SCrawler.YouTube/Controls/VideoOptionsForm.vb index c3a9fc5..164c24a 100644 --- a/SCrawler.YouTube/Controls/VideoOptionsForm.vb +++ b/SCrawler.YouTube/Controls/VideoOptionsForm.vb @@ -20,6 +20,7 @@ Imports UMTypes = SCrawler.Plugin.UserMediaTypes Imports ADB = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons Namespace API.YouTube.Controls Friend Class VideoOptionsForm : Implements IDesignXMLContainer + Friend Event AudioIndexChanged As EventHandler #Region "Declarations" Private MyView As FormView Private ReadOnly MyFieldsChecker As FieldsChecker @@ -29,6 +30,7 @@ Namespace API.YouTube.Controls Private Const ControlsRow As Integer = 7 Private ReadOnly Property CNT_PROCESSOR As TableControlsProcessor Friend Property MyContainer As YouTubeMediaContainerBase + Private ReadOnly AVControls As List(Of VideoOption) Private Initialization As Boolean = True Private ReadOnly InheritsFromContainer As Boolean Private ReadOnly M3U8Files As List(Of SFile) @@ -63,6 +65,7 @@ Namespace API.YouTube.Controls InitializeComponent() M3U8Files = New List(Of SFile) MyContainer = Container + AVControls = New List(Of VideoOption) CNT_PROCESSOR = New TableControlsProcessor(TP_CONTROLS) Me.InheritsFromContainer = InheritsFromContainer MyFieldsChecker = New FieldsChecker @@ -210,6 +213,7 @@ Namespace API.YouTube.Controls MyView.DisposeIfReady() MyFieldsChecker.DisposeIfReady() M3U8Files.Clear() + AVControls.ListClearDispose End Sub #End Region #Region "Refill" @@ -218,6 +222,7 @@ Namespace API.YouTube.Controls Dim h% = -1 Dim rh% = IIf(MyContainer.HasElements, 60, 25) Dim s As New Size(Width, h) + Dim codecW% = 0 Dim CalculateSize As Action(Of Integer) = Sub(ByVal InitHeight As Integer) With TP_MAIN.RowStyles(ControlsRow) : .SizeType = SizeType.Absolute : .Height = InitHeight : End With @@ -230,7 +235,7 @@ Namespace API.YouTube.Controls Dim __contentType As UMTypes = IIf(OPT_VIDEO.Checked, UMTypes.Video, UMTypes.Audio) With TP_CONTROLS If .Controls.Count > 0 Then - For Each cnt As Control In .Controls : cnt.Dispose() : Next + 'For Each cnt As Control In .Controls : cnt.Dispose() : Next .Controls.Clear() End If .RowStyles.Clear() @@ -239,14 +244,29 @@ Namespace API.YouTube.Controls With MyContainer Dim audio As MediaObject = Nothing If __contentType = UMTypes.Video Then audio = .SelectedAudio - Dim data As IEnumerable(Of Control) + Dim data As IEnumerable(Of Control) = Nothing If .HasElements Then data = .Elements.Select(Function(ee) New MediaItem(ee, True) With {.Dock = DockStyle.Fill, .Checked = ee.Checked, .UseCookies = UseCookies}) Else - data = (From m As MediaObject In .Self.MediaObjects - Where m.Type = __contentType Or (__contentType = UMTypes.Video And m.Type = YouTubeMediaContainerBase.TypeAudioVideo) - Select New VideoOption(m, audio) With {.Dock = DockStyle.Fill, .Checked = m.Index = MyContainer.SelectedVideoIndex}) + If AVControls.Count = 0 Then + AVControls.AddRange(From m As MediaObject In .Self.MediaObjects + Where m.Type = UMTypes.Audio Or m.Type = UMTypes.Video Or m.Type = YouTubeMediaContainerBase.TypeAudioVideo + Select New VideoOption(m, audio) With { + .Dock = DockStyle.Fill, + .Checked = m.Index = If(m.Type = UMTypes.Audio, MyContainer.SelectedAudioIndex, MyContainer.SelectedVideoIndex) + }) + If AVControls.Count > 0 Then + AVControls.ForEach(Sub(ByVal av As VideoOption) + AddHandler av.CheckedChanged, AddressOf VideoOption_CheckedChanged + av.SetAudioChangingHandler(Me) + End Sub) + End If + End If + 'data = (From m As MediaObject In .Self.MediaObjects + ' Where m.Type = __contentType Or (__contentType = UMTypes.Video And m.Type = YouTubeMediaContainerBase.TypeAudioVideo) + ' Select New VideoOption(m, audio) With {.Dock = DockStyle.Fill, .Checked = m.Index = MyContainer.SelectedVideoIndex}) + If AVControls.Count > 0 Then data = AVControls.Where(Function(av) av.ContentType = __contentType) End If If data.ListExists Then @@ -312,6 +332,16 @@ Namespace API.YouTube.Controls End With End If + If Not .HasElements Then + codecW = AVControls.Max(Function(av) av.CodecLength) + Dim baseW% = DirectCast(TP_CONTROLS.Controls(0), VideoOption).CodecLengthBase + If baseW + codecW + 12 + 30 > Width Then + s.Width += Math.Abs((Size.Width - baseW - codecW - 12 - 30)) + 'MinimumSize = s + 'Size = s + End If + End If + With TP_CONTROLS .PerformLayout() .Select() @@ -332,6 +362,16 @@ Namespace API.YouTube.Controls Private Sub MediaItem_CheckedChanged(ByVal Sender As MediaItem, ByVal Container As IYouTubeMediaContainer) ControlInvokeFast(TP_CONTROLS, Sub() Container.Checked = Sender.Checked, EDP.None) End Sub + Private Sub VideoOption_CheckedChanged(ByVal Sender As Object, ByVal e As EventArgs) + If Not Sender Is Nothing Then + With DirectCast(Sender, VideoOption) + If .MyMedia.Type = UMTypes.Audio And .Checked Then + MyContainer.SelectedAudioIndex = .MyMedia.Index + RaiseEvent AudioIndexChanged(Sender, e) + End If + End With + End If + End Sub Private Sub MediaItem_BeforeOpenEditor(ByVal Sender As MediaItem, ByVal Container As IYouTubeMediaContainer) MediaItem_BeforeOpenEditorImpl(Sender, Container, False) End Sub diff --git a/SCrawler.YouTube/Declarations.vb b/SCrawler.YouTube/Declarations.vb index b4d3591..dbf5dab 100644 --- a/SCrawler.YouTube/Declarations.vb +++ b/SCrawler.YouTube/Declarations.vb @@ -56,6 +56,7 @@ Namespace API.YouTube Friend ReadOnly TimeToStringProvider As IFormatProvider = New TimeToStringConverter Friend ReadOnly TimeToStringProviderH As IFormatProvider = New TimeToStringConverter(True) Friend ReadOnly TitleHtmlConverter As Func(Of String, String) = Function(Input) Input.StringRemoveWinForbiddenSymbols().StringTrim() + Friend ReadOnly MediaObjParseLang As RParams = RParams.DM("(?i)^(.+?)(?:\s*-\s*( .+?))?(?=\s*(?:,\s*)?(?:low|medium|high|DRC)\b|$)", 0, EDP.ReturnValue) Friend ReadOnly ProgressProvider As IMyProgressNumberProvider = MyProgressNumberProvider.Percentage Friend ReadOnly DownloadProgressPattern As RParams = RParams.DMS("\[download\]\s*([\d\.,]+)", 1, EDP.ReturnValue) Public ReadOnly TrueUrlRegEx As RParams = RParams.DM(Base.YouTubeFunctions.TrueUrlPattern, 0, EDP.ReturnValue) @@ -79,11 +80,12 @@ Namespace API.YouTube Return f End If End Function - Public Function CreateYtdlpProgressHandler(ByVal Progress As MyProgress, ByVal Token As Threading.CancellationToken, ByVal Batch As BatchExecutor) As DataReceivedEventHandler + Public Function CreateYtdlpProgressHandler(ByVal Progress As MyProgress, ByVal Token As Threading.CancellationToken, ByVal Batch As BatchExecutor, + Optional ByVal ValCalc As Func(Of Double, Double) = Nothing) As DataReceivedEventHandler Return Sub(ByVal Sender As Object, ByVal e As DataReceivedEventArgs) If Not e.Data.IsEmptyString Then Dim v# = AConvert(Of Double)(RegexReplace(e.Data, DownloadProgressPattern), NumberProvider, -1) - If v >= 0 Then Progress.Value = v : Progress.Perform(0) + If v >= 0 Then v = If(ValCalc Is Nothing, v, ValCalc.Invoke(v)) : Progress.Value = v : Progress.Perform(0) If Token.IsCancellationRequested Then Batch.Kill() End If End Sub diff --git a/SCrawler.YouTube/My Project/AssemblyInfo.vb b/SCrawler.YouTube/My Project/AssemblyInfo.vb index d5791bc..f8aa1f4 100644 --- a/SCrawler.YouTube/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTube/My Project/AssemblyInfo.vb @@ -13,7 +13,7 @@ Imports System.Runtime.InteropServices - + @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb index 226de9a..2d41908 100644 --- a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb +++ b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb @@ -1866,6 +1866,12 @@ Namespace API.YouTube.Objects obj.FPS = AConvert(Of Double)(ee.Value("fps"), NumberProvider, -1) obj.Bitrate = AConvert(Of Double)(ee.Value("tbr"), NumberProvider, -1) obj.Protocol = ee.Value("protocol") + obj.FormatNote = ee.Value("format_note") + If Not obj.FormatNote.IsEmptyString Then + obj.FormatNoteClean = RegexReplace(obj.FormatNote, MediaObjParseLang) + If Not obj.FormatNoteClean.IsEmptyString Then obj.FormatNoteClean = obj.FormatNoteClean.StringTrimEnd({CChar(" "), CChar("-"), CChar(",")}) + End If + obj.Language = ee.Value("language") If Not obj.Protocol.IsEmptyString Then obj.Protocol = obj.Protocol.Split("_").FirstOrDefault nValue = AConvert(Of Double)(ee.Value("filesize"), NumberProvider, -1) If nValue > 0 Then obj.Size = (nValue / 1024).RoundVal(2) @@ -1886,24 +1892,30 @@ Namespace API.YouTube.Objects obj.Codec = sValue.Split(".").First Else Dim fd As Boolean = False - sValue = ee.Value("format_note") - If Not sValue.IsEmptyString Then - With ListAddList(Nothing, sValue.Split(","), CType(Function(v) CStr(v).StringToLower.StringTrim, Func(Of Object, Object)), EDP.ReturnValue) - If .ListContains({"high", "low"}) Then - obj.Type = UMTypes.Audio - obj.Codec = ee.Value("ext") - If obj.Protocol.StringToLower.StartsWith("m3u8") Then obj.Protocol = "m3u8" - If obj.Bitrate <= 0 Then obj.Bitrate = IIf(.Contains("high"), 129, 53) - If obj.Size <= 0 Then obj.Size = 1 - fd = True - End If - End With + If Not obj.FormatQuality.IsEmptyString Or ee.Value("resolution") = "audio only" Then + obj.Type = UMTypes.Audio + obj.Codec = ee.Value("ext") + If obj.Protocol.StringToLower.StartsWith("m3u8") Then obj.Protocol = "m3u8" + If obj.Bitrate <= 0 Then obj.Bitrate = IIf(obj.FormatQuality = "high", 129, 53) + If obj.Size <= 0 Then obj.Size = 1 + fd = True End If If Not fd Then Continue For End If MediaObjects.Add(obj) Next MediaObjects.RemoveAll(Function(m) (m.Type = UMTypes.Video And (m.Width <= 0 Or m.Height <= 0)) Or m.URL.IsEmptyString) + + With MyYouTubeSettings + If .DefaultAudioLanguage_ExcludeAI Or .DefaultAudioLanguage_ExcludeNonDefault Then + If .DefaultAudioLanguage_ExcludeAI.Value AndAlso MediaObjects.LongCount(Function(m) m.Type = UMTypes.Audio And Not m.IsAI) > 0 Then _ + MediaObjects.RemoveAll(Function(m) m.Type = UMTypes.Audio And m.IsAI) + If MediaObjects.Count > 0 AndAlso .DefaultAudioLanguage_ExcludeNonDefault.Value AndAlso + MediaObjects.LongCount(Function(m) m.Type = UMTypes.Audio And m.IsDefaultOrOriginal) > 0 Then _ + MediaObjects.RemoveAll(Function(m) m.Type = UMTypes.Audio And Not m.IsDefaultOrOriginal) + End If + End With + Dim DupRemover As Action(Of UMTypes) = Sub(ByVal t As UMTypes) Const webm$ = "webm" diff --git a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb index 420b8b3..402beba 100644 --- a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb @@ -13,7 +13,7 @@ Imports System.Runtime.InteropServices - + @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/API/Base/UserDataBase.vb b/SCrawler/API/Base/UserDataBase.vb index cd2900f..be07de7 100644 --- a/SCrawler/API/Base/UserDataBase.vb +++ b/SCrawler/API/Base/UserDataBase.vb @@ -28,6 +28,7 @@ Namespace API.Base Friend MustInherit Class UserDataBase : Implements IUserData, IPluginContentProvider, IThrower Friend Const UserFileAppender As String = "User" Friend Const PostTextSpecialFolderDefault As String = "txt" + Friend Const PostTextTEXTSpecialFolderDefault As String = "Text" #Region "Events" Private ReadOnly UserUpdatedEventHandlers As List(Of IUserData.UserUpdatedEventHandler) Friend Custom Event UserUpdated As IUserData.UserUpdatedEventHandler Implements IUserData.UserUpdated @@ -1876,8 +1877,13 @@ stxt: Continue For End If End If + If Not fTxt.Path.StartsWith(MyDir) Then fTxt.Path = MyDir v.PostTextFileSpecialFolder = DownloadTextSpecialFolder - If DownloadTextSpecialFolder Then fTxt.Path = $"{fTxt.Path.StringTrimEnd("\")}\{PostTextSpecialFolderDefault}" + If v.Type = UTypes.Text Then + fTxt.Path = $"{fTxt.Path.StringTrimEnd("\")}\{PostTextTEXTSpecialFolderDefault}" + ElseIf DownloadTextSpecialFolder Then + fTxt.Path = $"{fTxt.Path.StringTrimEnd("\")}\{PostTextSpecialFolderDefault}" + End If fTxt.Extension = "txt" v.PostTextFile = TextSaver.SaveTextToFile(v.PostText, fTxt,,, Settings.FeedShowTextPosts_LogErrors_E) If Not v.PostTextFile.Exists Then Throw New ArgumentNullException("Text", "Error downloading text") With {.HelpLink = 10} diff --git a/SCrawler/API/Base/YTDLP.vb b/SCrawler/API/Base/YTDLP.vb index 8af5f08..49b8bc3 100644 --- a/SCrawler/API/Base/YTDLP.vb +++ b/SCrawler/API/Base/YTDLP.vb @@ -14,8 +14,8 @@ Namespace API.Base.YTDLP Friend Sub New(ByVal _Token As Threading.CancellationToken, Optional ByVal __MainProcessName As String = Nothing, Optional ByVal WorkingDir As SFile = Nothing) MyBase.New(_Token, __MainProcessName.IfNullOrEmpty(Settings.YtdlpFile.File.Name), WorkingDir.IfNullOrEmpty(Settings.YtdlpFile.File)) End Sub - Friend Sub SetProgress(ByVal Progress As MyProgress) - If Not Progress Is Nothing Then AddHandler OutputDataReceived, CreateYtdlpProgressHandler(Progress, Token, Me) + Friend Sub SetProgress(ByVal Progress As MyProgress, Optional ByVal ValCalc As Func(Of Double, Double) = Nothing) + If Not Progress Is Nothing Then AddHandler OutputDataReceived, CreateYtdlpProgressHandler(Progress, Token, Me, ValCalc) End Sub Friend Sub CreateFileExchanger(ByVal cache As CacheKeeper, Optional ByVal root As SFile = Nothing) FileExchanger = cache.NewInstance(Of BatchFileExchanger)(root, EDP.ReturnValue) diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb index 065dedc..bebe40f 100644 --- a/SCrawler/API/Instagram/UserData.vb +++ b/SCrawler/API/Instagram/UserData.vb @@ -727,7 +727,7 @@ Namespace API.Instagram processGetResponse = False Case Sections.SavedPosts ChangeResponserMode(False) - EndCursor = SavedPostsDownload(String.Empty, Token) + EndCursor = DownloadSavedPosts(Cursor, Token) HasNextPage = Not EndCursor.IsEmptyString MySiteSettings.TooManyRequests(False) ThrowAny(Token) @@ -965,7 +965,7 @@ NextPageBlock: End Try End Sub ''' Cursor - Private Function SavedPostsDownload(ByVal Cursor As String, ByVal Token As CancellationToken) As String + Private Function DownloadSavedPosts(ByVal Cursor As String, ByVal Token As CancellationToken) As String Dim URL$ = $"https://www.instagram.com/api/v1/feed/saved/posts/?max_id={Cursor}" Dim HasNextPage As Boolean = False Dim NextCursor$ = String.Empty diff --git a/SCrawler/API/OnlyFans/UserData.vb b/SCrawler/API/OnlyFans/UserData.vb index 4e93dd4..2d302f1 100644 --- a/SCrawler/API/OnlyFans/UserData.vb +++ b/SCrawler/API/OnlyFans/UserData.vb @@ -165,7 +165,11 @@ Namespace API.OnlyFans If IsSavedPosts Then path = $"/api2/v2/posts/bookmarks/all/?format=infinite&limit=10&offset={Cursor}" Else - path = $"/api2/v2/users/{ID}/posts/medias?limit=50&order=publish_date_desc&skip_users=all&format=infinite&counters=1" + If DownloadTextPosts And DownloadText Then + path = $"/api2/v2/users/{ID}/posts?limit=10&order=publish_date_desc&skip_users=all&format=infinite&pinned=0" + Else + path = $"/api2/v2/users/{ID}/posts/medias?limit=50&order=publish_date_desc&skip_users=all&format=infinite&counters=1" + End If If Not Cursor.IsEmptyString Then path &= $"&counters=0&beforePublishTime={Cursor}" Else path &= "&counters=1" End If @@ -500,7 +504,7 @@ Namespace API.OnlyFans ID = j.Value("id") UserSiteNameUpdate(j.Value("name")) Dim descr$ = j.Value("about") - If Not descr.IsEmptyString Then descr = descr.Replace(brTag, String.Empty) + If Not descr.IsEmptyString Then descr = descr.Replace(brTag, vbCrLf) UserDescriptionUpdate(descr) SimpleDownloadAvatar(j.Value("avatar")) SimpleDownloadAvatar(j.Value("header")) diff --git a/SCrawler/API/Xhamster/UserData.vb b/SCrawler/API/Xhamster/UserData.vb index e51497d..4625073 100644 --- a/SCrawler/API/Xhamster/UserData.vb +++ b/SCrawler/API/Xhamster/UserData.vb @@ -809,7 +809,7 @@ Namespace API.Xhamster Using ytdlp As New YTDLP.YTDLPBatch(Token,, DestinationFile) ytdlp.Encoding = Settings.CMDEncoding ytdlp.CreateFileExchanger(GetCache) - If IsSingleObjectDownload Then ytdlp.SetProgress(Progress) + If IsSingleObjectDownload Then ytdlp.SetProgress(Progress, Function(v) v / 100) ytdlp.Execute(cmd) End Using Return DestinationFile diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index 73ec001..b6426ec 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -13,7 +13,7 @@ Imports System.Runtime.InteropServices - + @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + +