geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

2024.4.8.0

Browse Source 
YT
MusicPlaylistsForm, VideoOptionsForm: add audio bitrate option
MediaItem: update type icon; update confirmation dialog for deleting non-single object
Track: update extension
PlayList: update 'ToString' information for 'MediaItem'
YouTubeMediaContainerBase: add size recalculation; add audio bitrate change; embed thumbnail in the extracted 'mp3' as cover art; update 'DownloadCommand' function; include elements' files in XML for non-single items; update 'Delete' function; update files handling; include generated playlists and cover file in the file list
YouTubeSettings: add properties 'DefaultAudioEmbedThumbnail_ExtractedFiles', 'DefaultAudioBitrate', 'DefaultAudioBitrate_crf'
Exclude 'drc' from parsing results
Fix incorrect file reference when the yt-dlp.exe has a different name

SCrawler
Base.Declarations: hide 'TokenRefreshIntervalProvider' error
Base.DeclaredNames: remove 'Header_FB_FRIENDLY_NAME' const (use 'API.Instagram.UserData.GQL')
Base.M3U8Base: add 'SkipBroken' argument
Base.UserDataBase: add size recalculation (STD)
Base.SiteSettingsBase: add 'SettingsVersion' property
TDownloader: delete 'RenameOldFileNames' function
SiteEditorForm: remove begin/end update of global settings when updating
MainFrame: update 'BTT_DOWN_SPEC' tooltip
SettingsHostCollection, SettingsHost: move site settings to a personal setting file (delete these settings from the global settings file)
DownloadGroupCollection: remove data update during initialization and reindexing
SettingsCLS: add 'SettingsVersion' property
Add hidden controls
API.JustForFans: change m3u8 parsing and downloading algo; remove 'CancellationToken' from m3u8 (replace with 'IThrower')
API.Facebook: add option 'RequestsWaitTimer_Any'; add internal option 'DownloadData_Impl'; update GDL names and tokens references; add wait timers
API.Threads: add option 'RequestsWaitTimer_Any'; add internal option 'DownloadData_Impl'; update GDL names and tokens references; add wait timers
API.Instagram: ADD 'GDL' SUPPORT; add 'UpdateWwwClaim' to 'Declarations.UpdateResponser' and 'UserData'; add additional 'HH_IG_WWW_CLAIM' properties; add 'RequestsWaitTimer_Any' property; add tooltips for timer controls; update 'LastRequests' environment; update information about requests on the label in the settings form; update reels downloading function
This commit is contained in:
Andy
2024-04-08 07:00:52 +03:00
parent 718eccc3c3
commit 5f90bf6a99
43 changed files with 1811 additions and 632 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
SCrawler.YouTube/Base/Structures.vb
View File

@@ -81,6 +81,7 @@ Namespace API.YouTube.Base
Public Structure MediaObject : Implements IIndexable, IComparable(Of MediaObject)
Public Type As Plugin.UserMediaTypes
Public ID As String
Public ID_DRC As Boolean
Public Extension As String
Public Width As Integer
Public Height As Integer
@@ -110,10 +111,14 @@ Namespace API.YouTube.Base
End Function
Private Function CompareTo(ByVal Other As MediaObject) As Integer Implements IComparable(Of MediaObject).CompareTo
If Type = Other.Type Then
If Width.CompareTo(Other.Width) = 0 Then
Return Size.CompareTo(Other.Size) * -1
If ID_DRC.CompareTo(Other.ID_DRC) = 0 Then
If Width.CompareTo(Other.Width) = 0 Then
Return Size.CompareTo(Other.Size) * -1
Else
Return Width.CompareTo(Other.Width) * -1
End If
Else
Return Width.CompareTo(Other.Width) * -1
Return ID_DRC.CompareTo(Other.ID_DRC)
End If
Else
Return CInt(Type).CompareTo(CInt(Other.Type))

2
SCrawler.YouTube/Base/YouTubeFunctions.vb
View File

@@ -174,7 +174,7 @@ Namespace API.YouTube.Base
ByVal ObjType As YouTubeMediaType, ByVal ChannelTab As YouTubeChannelTab,
ByVal IsMusic As Boolean, ByVal UrlAsIs As Boolean) As Boolean
Try
Dim command$ = "yt-dlp --write-info-json --skip-download"
Dim command$ = $"{YTDLP_NAME} --write-info-json --skip-download"
command.StringAppend(GetCookiesCommand(UseCookies, CookiesFile), " ")
If DateAfter.HasValue Then command.StringAppend($"--dateafter {DateAfter.Value:yyyyMMdd}", " ")
If DateBefore.HasValue Then command.StringAppend($"--datebefore {DateBefore.Value:yyyyMMdd}", " ")

10
SCrawler.YouTube/Base/YouTubeSettings.vb
View File

@@ -38,6 +38,7 @@ Namespace API.YouTube.Base
<Browsable(False)> Friend ReadOnly Property DownloadLocations As DownloadLocationsCollection
<Browsable(False)> Friend ReadOnly Property PlaylistsLocations As DownloadLocationsCollection
<Browsable(False)> Public Overridable Property AccountName As String
<Browsable(False), XMLVV(0)> Private ReadOnly Property SettingsVersion As XMLValue(Of Integer)
#Region "Environment"
#Region "Programs"
<Browsable(True), GridVisible(False), XMLVN({"Environment"}), Category("Environment programs"), DisplayName("Path to yt-dlp.exe"),
@@ -379,6 +380,15 @@ Namespace API.YouTube.Base
<Browsable(True), GridVisible, XMLVN({"DefaultsAudio"}, True), Category("Defaults Audio"), DisplayName("Embed thumbnail"),
Description("Embed thumbnail in the audio as cover art. Default: true.")>
Public ReadOnly Property DefaultAudioEmbedThumbnail As XMLValue(Of Boolean)
<Browsable(True), GridVisible, XMLVN({"DefaultsAudio"}, True), Category("Defaults Audio"), DisplayName("Embed thumbnail (extracted files)"),
Description("Embed thumbnail in the extracted (additional file ('mp3' only)) audio as cover art. Default: true.")>
Public ReadOnly Property DefaultAudioEmbedThumbnail_ExtractedFiles As XMLValue(Of Boolean)
<Browsable(True), GridVisible, XMLVN({"DefaultsAudio"}, -1), Category("Defaults Audio"), DisplayName("Bitrate"),
Description("Default audio bitrate if you want to change it during download. -1 to disable. Default: -1.")>
Public ReadOnly Property DefaultAudioBitrate As XMLValue(Of Integer)
<Browsable(True), GridVisible, XMLVN({"DefaultsAudio"}, 20), Category("Defaults Audio"), DisplayName("Bitrate: ffmpeg crf"),
Description("This is the ffmpeg argument. Change it only if you know what you're doing. Default: 20.")>
Public ReadOnly Property DefaultAudioBitrate_crf As XMLValue(Of Integer)
#Region "Music"
<Browsable(True), GridVisible, XMLVN({"Playlists"}, True), Category("Music"), DisplayName("Create M3U8"),
Description("Create M3U8 playlist for music. Default: true.")>

61
SCrawler.YouTube/Controls/MusicPlaylistsForm.Designer.vb generated
View File

@@ -52,6 +52,7 @@ Namespace API.YouTube.Controls
Dim ActionButton15 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton16 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton17 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton18 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim TT_MAIN As System.Windows.Forms.ToolTip
Me.BTT_DOWN = New System.Windows.Forms.Button()
Me.BTT_CANCEL = New System.Windows.Forms.Button()
@@ -66,6 +67,7 @@ Namespace API.YouTube.Controls
Me.CH_DOWN_LYRICS = New System.Windows.Forms.CheckBox()
Me.TXT_OUTPUT_PATH = New PersonalUtilities.Forms.Controls.ComboBoxExtended()
Me.CMB_PLS = New PersonalUtilities.Forms.Controls.ComboBoxExtended()
Me.TXT_AUDIO_BITRATE = New PersonalUtilities.Forms.Controls.TextBoxExtended()
TP_MAIN = New System.Windows.Forms.TableLayoutPanel()
TP_BUTTONS = New System.Windows.Forms.TableLayoutPanel()
TP_PLS = New System.Windows.Forms.TableLayoutPanel()
@@ -92,6 +94,7 @@ Namespace API.YouTube.Controls
CType(Me.TXT_SUBS, System.ComponentModel.ISupportInitialize).BeginInit()
CType(Me.TXT_OUTPUT_PATH, System.ComponentModel.ISupportInitialize).BeginInit()
CType(Me.CMB_PLS, System.ComponentModel.ISupportInitialize).BeginInit()
CType(Me.TXT_AUDIO_BITRATE, System.ComponentModel.ISupportInitialize).BeginInit()
Me.SuspendLayout()
'
'TP_MAIN
@@ -106,10 +109,10 @@ Namespace API.YouTube.Controls
TP_MAIN.Margin = New System.Windows.Forms.Padding(0)
TP_MAIN.Name = "TP_MAIN"
TP_MAIN.RowCount = 3
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 112.0!))
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 140.0!))
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
TP_MAIN.Size = New System.Drawing.Size(434, 289)
TP_MAIN.Size = New System.Drawing.Size(434, 317)
TP_MAIN.TabIndex = 0
'
'TP_BUTTONS
@@ -121,14 +124,14 @@ Namespace API.YouTube.Controls
TP_BUTTONS.Controls.Add(Me.BTT_DOWN, 1, 0)
TP_BUTTONS.Controls.Add(Me.BTT_CANCEL, 2, 0)
TP_BUTTONS.Dock = System.Windows.Forms.DockStyle.Fill
TP_BUTTONS.Location = New System.Drawing.Point(0, 264)
TP_BUTTONS.Location = New System.Drawing.Point(0, 292)
TP_BUTTONS.Margin = New System.Windows.Forms.Padding(0)
TP_BUTTONS.Name = "TP_BUTTONS"
TP_BUTTONS.RowCount = 1
TP_BUTTONS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_BUTTONS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
TP_BUTTONS.Size = New System.Drawing.Size(434, 25)
TP_BUTTONS.TabIndex = 2
TP_BUTTONS.TabIndex = 1
'
'BTT_DOWN
'
@@ -156,7 +159,7 @@ Namespace API.YouTube.Controls
'SPLITTER_MAIN
'
Me.SPLITTER_MAIN.Dock = System.Windows.Forms.DockStyle.Fill
Me.SPLITTER_MAIN.Location = New System.Drawing.Point(3, 115)
Me.SPLITTER_MAIN.Location = New System.Drawing.Point(3, 143)
Me.SPLITTER_MAIN.Name = "SPLITTER_MAIN"
'
'SPLITTER_MAIN.Panel1
@@ -272,18 +275,20 @@ Namespace API.YouTube.Controls
TP_SETTINGS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_SETTINGS.Controls.Add(TP_FORMATS, 0, 1)
TP_SETTINGS.Controls.Add(TP_LYRICS, 0, 0)
TP_SETTINGS.Controls.Add(Me.TXT_OUTPUT_PATH, 0, 2)
TP_SETTINGS.Controls.Add(Me.CMB_PLS, 0, 3)
TP_SETTINGS.Controls.Add(Me.TXT_OUTPUT_PATH, 0, 3)
TP_SETTINGS.Controls.Add(Me.CMB_PLS, 0, 4)
TP_SETTINGS.Controls.Add(Me.TXT_AUDIO_BITRATE, 0, 2)
TP_SETTINGS.Dock = System.Windows.Forms.DockStyle.Fill
TP_SETTINGS.Location = New System.Drawing.Point(0, 0)
TP_SETTINGS.Margin = New System.Windows.Forms.Padding(0)
TP_SETTINGS.Name = "TP_SETTINGS"
TP_SETTINGS.RowCount = 4
TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 25.0!))
TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 25.0!))
TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 25.0!))
TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 25.0!))
TP_SETTINGS.Size = New System.Drawing.Size(434, 112)
TP_SETTINGS.RowCount = 5
TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 20.0!))
TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 20.0!))
TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 20.0!))
TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 20.0!))
TP_SETTINGS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 20.0!))
TP_SETTINGS.Size = New System.Drawing.Size(434, 140)
TP_SETTINGS.TabIndex = 1
'
'TP_FORMATS
@@ -302,7 +307,7 @@ Namespace API.YouTube.Controls
TP_FORMATS.RowCount = 1
TP_FORMATS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_FORMATS.Size = New System.Drawing.Size(434, 28)
TP_FORMATS.TabIndex = 1
TP_FORMATS.TabIndex = 5
'
'TXT_FORMATS_ADDIT
'
@@ -371,7 +376,7 @@ Namespace API.YouTube.Controls
TP_LYRICS.RowCount = 1
TP_LYRICS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_LYRICS.Size = New System.Drawing.Size(434, 28)
TP_LYRICS.TabIndex = 0
TP_LYRICS.TabIndex = 6
'
'TXT_SUBS
'
@@ -462,7 +467,7 @@ Namespace API.YouTube.Controls
Me.TXT_OUTPUT_PATH.Columns.Add(ListColumn1)
Me.TXT_OUTPUT_PATH.Columns.Add(ListColumn2)
Me.TXT_OUTPUT_PATH.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_OUTPUT_PATH.Location = New System.Drawing.Point(3, 59)
Me.TXT_OUTPUT_PATH.Location = New System.Drawing.Point(3, 87)
Me.TXT_OUTPUT_PATH.Name = "TXT_OUTPUT_PATH"
Me.TXT_OUTPUT_PATH.Size = New System.Drawing.Size(428, 22)
Me.TXT_OUTPUT_PATH.TabIndex = 2
@@ -505,23 +510,39 @@ Namespace API.YouTube.Controls
Me.CMB_PLS.CaptionVisible = True
Me.CMB_PLS.CaptionWidth = 50.0R
Me.CMB_PLS.Dock = System.Windows.Forms.DockStyle.Fill
Me.CMB_PLS.Location = New System.Drawing.Point(3, 87)
Me.CMB_PLS.Location = New System.Drawing.Point(3, 115)
Me.CMB_PLS.Name = "CMB_PLS"
Me.CMB_PLS.Size = New System.Drawing.Size(428, 22)
Me.CMB_PLS.TabIndex = 3
Me.CMB_PLS.TextBoxBorderStyle = System.Windows.Forms.BorderStyle.FixedSingle
'
'TXT_AUDIO_BITRATE
'
ActionButton18.BackgroundImage = CType(resources.GetObject("ActionButton18.BackgroundImage"), System.Drawing.Image)
ActionButton18.Name = "Clear"
ActionButton18.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
Me.TXT_AUDIO_BITRATE.Buttons.Add(ActionButton18)
Me.TXT_AUDIO_BITRATE.CaptionText = "Audio bitrate"
Me.TXT_AUDIO_BITRATE.CaptionToolTipEnabled = True
Me.TXT_AUDIO_BITRATE.CaptionToolTipText = "Default audio bitrate if you want to change it during download"
Me.TXT_AUDIO_BITRATE.CaptionWidth = 112.0R
Me.TXT_AUDIO_BITRATE.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_AUDIO_BITRATE.Location = New System.Drawing.Point(3, 59)
Me.TXT_AUDIO_BITRATE.Name = "TXT_AUDIO_BITRATE"
Me.TXT_AUDIO_BITRATE.Size = New System.Drawing.Size(428, 22)
Me.TXT_AUDIO_BITRATE.TabIndex = 4
'
'MusicPlaylistsForm
'
Me.AcceptButton = Me.BTT_DOWN
Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)
Me.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font
Me.CancelButton = Me.BTT_CANCEL
Me.ClientSize = New System.Drawing.Size(434, 289)
Me.ClientSize = New System.Drawing.Size(434, 317)
Me.Controls.Add(TP_MAIN)
Me.Icon = Global.SCrawler.My.Resources.SiteYouTube.YouTubeMusicIcon_32
Me.KeyPreview = True
Me.MinimumSize = New System.Drawing.Size(450, 328)
Me.MinimumSize = New System.Drawing.Size(450, 356)
Me.Name = "MusicPlaylistsForm"
Me.Text = "Albums"
TP_MAIN.ResumeLayout(False)
@@ -541,6 +562,7 @@ Namespace API.YouTube.Controls
CType(Me.TXT_SUBS, System.ComponentModel.ISupportInitialize).EndInit()
CType(Me.TXT_OUTPUT_PATH, System.ComponentModel.ISupportInitialize).EndInit()
CType(Me.CMB_PLS, System.ComponentModel.ISupportInitialize).EndInit()
CType(Me.TXT_AUDIO_BITRATE, System.ComponentModel.ISupportInitialize).EndInit()
Me.ResumeLayout(False)
End Sub
@@ -557,5 +579,6 @@ Namespace API.YouTube.Controls
Private WithEvents CH_DOWN_LYRICS As CheckBox
Private WithEvents TXT_OUTPUT_PATH As PersonalUtilities.Forms.Controls.ComboBoxExtended
Private WithEvents CMB_PLS As PersonalUtilities.Forms.Controls.ComboBoxExtended
Private WithEvents TXT_AUDIO_BITRATE As PersonalUtilities.Forms.Controls.TextBoxExtended
End Class
End Namespace

8
SCrawler.YouTube/Controls/MusicPlaylistsForm.resx
View File

@@ -504,6 +504,14 @@
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6LEtW/4flgYiLD1qeX0A
AAAASUVORK5CYII=
</value>
</data>
<data name="ActionButton18.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go
tbbB43rK5xSAQq1VYFtmeQBoqZTSreVZvgTknM8yyyjA/qodsDF9gspD2Bj6B+DH+NqzhQQAG+POMnSX
AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC
</value>
</data>
</root>

12
SCrawler.YouTube/Controls/MusicPlaylistsForm.vb
View File

@@ -19,6 +19,7 @@ Namespace API.YouTube.Controls
Friend Class MusicPlaylistsForm : Implements IDesignXMLContainer
#Region "Declarations"
Private MyView As FormView
Private ReadOnly MyFieldsChecker As FieldsChecker
Friend Property DesignXML As EContainer Implements IDesignXMLContainer.DesignXML
Private Property DesignXMLNodes As String() Implements IDesignXMLContainer.DesignXMLNodes
Private Property DesignXMLNodeName As String Implements IDesignXMLContainer.DesignXMLNodeName
@@ -48,6 +49,7 @@ Namespace API.YouTube.Controls
InitializeComponent()
M3U8Files = New List(Of SFile)
MyContainer = Container
MyFieldsChecker = New FieldsChecker
End Sub
#End Region
#Region "Form handlers"
@@ -61,6 +63,7 @@ Namespace API.YouTube.Controls
MyYouTubeSettings.DownloadLocations.PopulateComboBox(TXT_OUTPUT_PATH)
MyYouTubeSettings.PlaylistsLocations.PopulateComboBox(CMB_PLS,, True)
CMB_PLS.Text = MyYouTubeSettings.LatestPlaylistFile.Value
If MyYouTubeSettings.DefaultAudioBitrate > 0 Then TXT_AUDIO_BITRATE.Text = MyYouTubeSettings.DefaultAudioBitrate.Value
CMB_FORMATS.Items.AddRange(AvailableAudioFormats)
If MyYouTubeSettings.PlaylistFormSplitterDistance > 0 Then SPLITTER_MAIN.SplitterDistancePercentageSet(MyYouTubeSettings.PlaylistFormSplitterDistance)
@@ -113,6 +116,9 @@ Namespace API.YouTube.Controls
Text = .PlaylistTitle
End If
MyFieldsChecker.AddControl(Of Integer)(TXT_AUDIO_BITRATE, TXT_AUDIO_BITRATE.CaptionText, True)
MyFieldsChecker.EndLoaderOperations()
UpdateSizeText()
End With
RefillAddit()
@@ -120,7 +126,8 @@ Namespace API.YouTube.Controls
End Sub
Private Sub MusicPlaylistsForm_Closing(sender As Object, e As CancelEventArgs) Handles Me.Closing
MyYouTubeSettings.PlaylistFormSplitterDistance.Value = SPLITTER_MAIN.SplitterDistancePercentageGet
MyView.DisposeIfReady()
MyView.DisposeIfReady
MyFieldsChecker.DisposeIfReady
M3U8Files.Clear()
End Sub
Private Sub MusicPlaylistsForm_KeyDown(sender As Object, e As KeyEventArgs) Handles Me.KeyDown
@@ -322,7 +329,7 @@ Namespace API.YouTube.Controls
Private Sub BTT_DOWN_Click(sender As Object, e As EventArgs) Handles BTT_DOWN.Click
If TXT_OUTPUT_PATH.IsEmptyString Then
MsgBoxE({"The output path cannot be null.", "Download music"}, vbCritical)
Else
ElseIf MyFieldsChecker.AllParamsOK Then
With DirectCast(MyContainer, YouTubeMediaContainerBase)
.OutputSubtitlesFormat = IIf(CH_DOWN_LYRICS.Checked, "LRC", String.Empty)
If Not TXT_SUBS.Checked Then .PostProcessing_OutputSubtitlesFormats.Clear()
@@ -331,6 +338,7 @@ Namespace API.YouTube.Controls
.AbsolutePath = TXT_OUTPUT_PATH.Checked
.File = TXT_OUTPUT_PATH.Text.CSFileP
.M3U8_PlaylistFiles = M3U8FilesFull
.OutputAudioBitrate = AConvert(Of Integer)(TXT_AUDIO_BITRATE.Text, -1)
If MyYouTubeSettings.OutputPathAutoChange Then MyYouTubeSettings.OutputPath.Value = .File
If MyDownloaderSettings.OutputPathAutoAddPaths Then MyYouTubeSettings.DownloadLocations.Add(.File, False)
If Not CMB_PLS.Text.IsEmptyString Then MyYouTubeSettings.PlaylistsLocations.Add(CMB_PLS.Text, False, True)

4
SCrawler.YouTube/Controls/VideoOption.vb
View File

@@ -27,6 +27,7 @@ Namespace API.YouTube.Controls
Friend Sub New(ByVal m As MediaObject, Optional ByVal SelectedAudio As MediaObject = Nothing)
Me.New
Const d$ = " " & ChrW(183) & " "
Const DRC$ = Objects.YouTubeMediaContainerBase.DRC
MyMedia = m
If m.Type = Plugin.UserMediaTypes.Audio Then
If m.Bitrate >= 320 Then
@@ -38,6 +39,7 @@ Namespace API.YouTube.Controls
End If
LBL_DEFINITION.Text = $"{m.Bitrate}k"
LBL_CODECS.Text = $"{m.Extension} {d} {m.Codec} {d} {m.Bitrate}k"
If Not m.ID.IsEmptyString AndAlso m.ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $" {d} DRC"
Else
If m.Height >= 1440 Then
LBL_DEFINITION_INFO.Text = "Ultra High Definition"
@@ -53,7 +55,9 @@ Namespace API.YouTube.Controls
LBL_DEFINITION.Text = $"{m.Height}p"
LBL_CODECS.Text = $"{m.Extension.StringToUpper}{d}{m.Codec.StringToUpper}{d}{m.FPS}fps{d}{m.Bitrate}k"
If Not m.Protocol.IsEmptyString Then LBL_CODECS.Text &= $" ({m.Protocol})"
If Not m.ID.IsEmptyString AndAlso m.ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $"{d}DRC"
If Not SelectedAudio.ID.IsEmptyString Then LBL_CODECS.Text &= $" / {SelectedAudio.Extension}{d}{SelectedAudio.Codec}{d}{SelectedAudio.Bitrate}k"
If Not SelectedAudio.ID.IsEmptyString AndAlso SelectedAudio.ID.StringToLower.Contains(DRC) Then LBL_CODECS.Text &= $"{d}DRC"
End If
Dim sv% = m.Size / 1024

265
SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb generated
View File

@@ -47,6 +47,7 @@ Namespace API.YouTube.Controls
Dim LBL_FORMAT As System.Windows.Forms.Label
Dim LBL_SUBS_FORMAT As System.Windows.Forms.Label
Dim TT_MAIN As System.Windows.Forms.ToolTip
Dim TP_FPS_BITRATE As System.Windows.Forms.TableLayoutPanel
Dim ActionButton7 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton8 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton9 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
@@ -57,6 +58,7 @@ Namespace API.YouTube.Controls
Dim ActionButton14 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton15 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton16 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton17 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Me.ICON_VIDEO = New System.Windows.Forms.PictureBox()
Me.LBL_TITLE = New System.Windows.Forms.Label()
Me.TP_HEADER_INFO_2 = New System.Windows.Forms.TableLayoutPanel()
@@ -71,6 +73,8 @@ Namespace API.YouTube.Controls
Me.OPT_VIDEO = New System.Windows.Forms.RadioButton()
Me.OPT_AUDIO = New System.Windows.Forms.RadioButton()
Me.LBL_AUDIO_CODEC = New System.Windows.Forms.Label()
Me.TXT_FPS = New PersonalUtilities.Forms.Controls.TextBoxExtended()
Me.TXT_AUDIO_BITRATE = New PersonalUtilities.Forms.Controls.TextBoxExtended()
Me.TP_HEADER_BASE = New System.Windows.Forms.TableLayoutPanel()
Me.TP_SUBS = New System.Windows.Forms.TableLayoutPanel()
Me.TXT_SUBS = New PersonalUtilities.Forms.Controls.TextBoxExtended()
@@ -80,7 +84,6 @@ Namespace API.YouTube.Controls
Me.CMB_FORMAT = New System.Windows.Forms.ComboBox()
Me.CMB_AUDIO_CODEC = New System.Windows.Forms.ComboBox()
Me.NUM_RES = New System.Windows.Forms.NumericUpDown()
Me.TXT_FPS = New PersonalUtilities.Forms.Controls.TextBoxExtended()
Me.TP_CONTROLS = New System.Windows.Forms.TableLayoutPanel()
Me.TXT_SUBS_ADDIT = New PersonalUtilities.Forms.Controls.TextBoxExtended()
Me.TXT_EXTRA_AUDIO_FORMATS = New PersonalUtilities.Forms.Controls.TextBoxExtended()
@@ -99,6 +102,7 @@ Namespace API.YouTube.Controls
LBL_FORMAT = New System.Windows.Forms.Label()
LBL_SUBS_FORMAT = New System.Windows.Forms.Label()
TT_MAIN = New System.Windows.Forms.ToolTip(Me.components)
TP_FPS_BITRATE = New System.Windows.Forms.TableLayoutPanel()
TP_HEADER.SuspendLayout()
CType(Me.ICON_VIDEO, System.ComponentModel.ISupportInitialize).BeginInit()
TP_HEADER_INFO.SuspendLayout()
@@ -112,13 +116,15 @@ Namespace API.YouTube.Controls
TP_PLS.SuspendLayout()
CType(Me.CMB_PLS, System.ComponentModel.ISupportInitialize).BeginInit()
TP_WHAT.SuspendLayout()
TP_FPS_BITRATE.SuspendLayout()
CType(Me.TXT_FPS, System.ComponentModel.ISupportInitialize).BeginInit()
CType(Me.TXT_AUDIO_BITRATE, System.ComponentModel.ISupportInitialize).BeginInit()
Me.TP_HEADER_BASE.SuspendLayout()
Me.TP_SUBS.SuspendLayout()
CType(Me.TXT_SUBS, System.ComponentModel.ISupportInitialize).BeginInit()
Me.TP_MAIN.SuspendLayout()
Me.TP_OPTIONS.SuspendLayout()
CType(Me.NUM_RES, System.ComponentModel.ISupportInitialize).BeginInit()
CType(Me.TXT_FPS, System.ComponentModel.ISupportInitialize).BeginInit()
CType(Me.TXT_SUBS_ADDIT, System.ComponentModel.ISupportInitialize).BeginInit()
CType(Me.TXT_EXTRA_AUDIO_FORMATS, System.ComponentModel.ISupportInitialize).BeginInit()
Me.SuspendLayout()
@@ -137,7 +143,7 @@ Namespace API.YouTube.Controls
TP_HEADER.Name = "TP_HEADER"
TP_HEADER.RowCount = 1
TP_HEADER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_HEADER.Size = New System.Drawing.Size(719, 63)
TP_HEADER.Size = New System.Drawing.Size(599, 63)
TP_HEADER.TabIndex = 0
'
'ICON_VIDEO
@@ -166,7 +172,7 @@ Namespace API.YouTube.Controls
TP_HEADER_INFO.RowCount = 2
TP_HEADER_INFO.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 50.0!))
TP_HEADER_INFO.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 50.0!))
TP_HEADER_INFO.Size = New System.Drawing.Size(589, 63)
TP_HEADER_INFO.Size = New System.Drawing.Size(469, 63)
TP_HEADER_INFO.TabIndex = 0
'
'LBL_TITLE
@@ -175,7 +181,7 @@ Namespace API.YouTube.Controls
Me.LBL_TITLE.Font = New System.Drawing.Font("Arial", 9.0!, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, CType(204, Byte))
Me.LBL_TITLE.Location = New System.Drawing.Point(3, 0)
Me.LBL_TITLE.Name = "LBL_TITLE"
Me.LBL_TITLE.Size = New System.Drawing.Size(583, 31)
Me.LBL_TITLE.Size = New System.Drawing.Size(463, 31)
Me.LBL_TITLE.TabIndex = 0
Me.LBL_TITLE.Text = "Video title"
Me.LBL_TITLE.TextAlign = System.Drawing.ContentAlignment.MiddleLeft
@@ -197,7 +203,7 @@ Namespace API.YouTube.Controls
Me.TP_HEADER_INFO_2.Name = "TP_HEADER_INFO_2"
Me.TP_HEADER_INFO_2.RowCount = 1
Me.TP_HEADER_INFO_2.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.TP_HEADER_INFO_2.Size = New System.Drawing.Size(589, 32)
Me.TP_HEADER_INFO_2.Size = New System.Drawing.Size(469, 32)
Me.TP_HEADER_INFO_2.TabIndex = 1
'
'ICON_CLOCK
@@ -244,7 +250,7 @@ Namespace API.YouTube.Controls
Me.LBL_URL.LinkColor = System.Drawing.Color.FromArgb(CType(CType(0, Byte), Integer), CType(CType(0, Byte), Integer), CType(CType(192, Byte), Integer))
Me.LBL_URL.Location = New System.Drawing.Point(115, 0)
Me.LBL_URL.Name = "LBL_URL"
Me.LBL_URL.Size = New System.Drawing.Size(471, 32)
Me.LBL_URL.Size = New System.Drawing.Size(351, 32)
Me.LBL_URL.TabIndex = 1
Me.LBL_URL.TabStop = True
Me.LBL_URL.Text = "https://www.youtube.com/watch?v=abcdefghijk"
@@ -258,14 +264,14 @@ Namespace API.YouTube.Controls
TP_FOOTER.Controls.Add(TP_OK_CANCEL, 0, 2)
TP_FOOTER.Controls.Add(TP_PLS, 0, 0)
TP_FOOTER.Dock = System.Windows.Forms.DockStyle.Fill
TP_FOOTER.Location = New System.Drawing.Point(6, 215)
TP_FOOTER.Location = New System.Drawing.Point(6, 243)
TP_FOOTER.Margin = New System.Windows.Forms.Padding(6, 3, 6, 3)
TP_FOOTER.Name = "TP_FOOTER"
TP_FOOTER.RowCount = 3
TP_FOOTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 33.33333!))
TP_FOOTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 33.33333!))
TP_FOOTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 33.33333!))
TP_FOOTER.Size = New System.Drawing.Size(709, 81)
TP_FOOTER.Size = New System.Drawing.Size(589, 81)
TP_FOOTER.TabIndex = 5
'
'TP_DESTINATION
@@ -281,7 +287,7 @@ Namespace API.YouTube.Controls
TP_DESTINATION.Name = "TP_DESTINATION"
TP_DESTINATION.RowCount = 1
TP_DESTINATION.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_DESTINATION.Size = New System.Drawing.Size(709, 27)
TP_DESTINATION.Size = New System.Drawing.Size(589, 27)
TP_DESTINATION.TabIndex = 0
'
'TXT_FILE
@@ -310,14 +316,14 @@ Namespace API.YouTube.Controls
Me.TXT_FILE.Location = New System.Drawing.Point(1, 1)
Me.TXT_FILE.Margin = New System.Windows.Forms.Padding(1)
Me.TXT_FILE.Name = "TXT_FILE"
Me.TXT_FILE.Size = New System.Drawing.Size(627, 22)
Me.TXT_FILE.Size = New System.Drawing.Size(507, 22)
Me.TXT_FILE.TabIndex = 0
Me.TXT_FILE.TextBoxBorderStyle = System.Windows.Forms.BorderStyle.FixedSingle
'
'BTT_BROWSE
'
Me.BTT_BROWSE.Dock = System.Windows.Forms.DockStyle.Fill
Me.BTT_BROWSE.Location = New System.Drawing.Point(632, 2)
Me.BTT_BROWSE.Location = New System.Drawing.Point(512, 2)
Me.BTT_BROWSE.Margin = New System.Windows.Forms.Padding(3, 2, 3, 2)
Me.BTT_BROWSE.Name = "BTT_BROWSE"
Me.BTT_BROWSE.Size = New System.Drawing.Size(74, 23)
@@ -341,13 +347,13 @@ Namespace API.YouTube.Controls
TP_OK_CANCEL.RowCount = 1
TP_OK_CANCEL.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_OK_CANCEL.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 27.0!))
TP_OK_CANCEL.Size = New System.Drawing.Size(709, 27)
TP_OK_CANCEL.Size = New System.Drawing.Size(589, 27)
TP_OK_CANCEL.TabIndex = 1
'
'BTT_DOWN
'
Me.BTT_DOWN.Dock = System.Windows.Forms.DockStyle.Fill
Me.BTT_DOWN.Location = New System.Drawing.Point(552, 2)
Me.BTT_DOWN.Location = New System.Drawing.Point(432, 2)
Me.BTT_DOWN.Margin = New System.Windows.Forms.Padding(3, 2, 3, 2)
Me.BTT_DOWN.Name = "BTT_DOWN"
Me.BTT_DOWN.Size = New System.Drawing.Size(74, 23)
@@ -359,7 +365,7 @@ Namespace API.YouTube.Controls
'
Me.BTT_CANCEL.DialogResult = System.Windows.Forms.DialogResult.Cancel
Me.BTT_CANCEL.Dock = System.Windows.Forms.DockStyle.Fill
Me.BTT_CANCEL.Location = New System.Drawing.Point(632, 2)
Me.BTT_CANCEL.Location = New System.Drawing.Point(512, 2)
Me.BTT_CANCEL.Margin = New System.Windows.Forms.Padding(3, 2, 3, 2)
Me.BTT_CANCEL.Name = "BTT_CANCEL"
Me.BTT_CANCEL.Size = New System.Drawing.Size(74, 23)
@@ -381,7 +387,7 @@ Namespace API.YouTube.Controls
TP_PLS.RowCount = 1
TP_PLS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_PLS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 27.0!))
TP_PLS.Size = New System.Drawing.Size(709, 27)
TP_PLS.Size = New System.Drawing.Size(589, 27)
TP_PLS.TabIndex = 2
'
'CMB_PLS
@@ -414,14 +420,14 @@ Namespace API.YouTube.Controls
Me.CMB_PLS.Location = New System.Drawing.Point(1, 1)
Me.CMB_PLS.Margin = New System.Windows.Forms.Padding(1)
Me.CMB_PLS.Name = "CMB_PLS"
Me.CMB_PLS.Size = New System.Drawing.Size(627, 22)
Me.CMB_PLS.Size = New System.Drawing.Size(507, 22)
Me.CMB_PLS.TabIndex = 0
Me.CMB_PLS.TextBoxBorderStyle = System.Windows.Forms.BorderStyle.FixedSingle
'
'BTT_PLS_BROWSE
'
Me.BTT_PLS_BROWSE.Dock = System.Windows.Forms.DockStyle.Fill
Me.BTT_PLS_BROWSE.Location = New System.Drawing.Point(632, 2)
Me.BTT_PLS_BROWSE.Location = New System.Drawing.Point(512, 2)
Me.BTT_PLS_BROWSE.Margin = New System.Windows.Forms.Padding(3, 2, 3, 2)
Me.BTT_PLS_BROWSE.Name = "BTT_PLS_BROWSE"
Me.BTT_PLS_BROWSE.Size = New System.Drawing.Size(74, 23)
@@ -434,20 +440,20 @@ Namespace API.YouTube.Controls
'
LB_SEP_1.Anchor = CType((System.Windows.Forms.AnchorStyles.Left Or System.Windows.Forms.AnchorStyles.Right), System.Windows.Forms.AnchorStyles)
LB_SEP_1.BackColor = System.Drawing.SystemColors.ControlDark
LB_SEP_1.Location = New System.Drawing.Point(6, 179)
LB_SEP_1.Location = New System.Drawing.Point(6, 207)
LB_SEP_1.Margin = New System.Windows.Forms.Padding(6, 0, 6, 0)
LB_SEP_1.Name = "LB_SEP_1"
LB_SEP_1.Size = New System.Drawing.Size(709, 1)
LB_SEP_1.Size = New System.Drawing.Size(589, 1)
LB_SEP_1.TabIndex = 3
'
'LB_SEP_2
'
LB_SEP_2.Anchor = CType((System.Windows.Forms.AnchorStyles.Left Or System.Windows.Forms.AnchorStyles.Right), System.Windows.Forms.AnchorStyles)
LB_SEP_2.BackColor = System.Drawing.SystemColors.ControlDark
LB_SEP_2.Location = New System.Drawing.Point(6, 209)
LB_SEP_2.Location = New System.Drawing.Point(6, 237)
LB_SEP_2.Margin = New System.Windows.Forms.Padding(6, 0, 6, 0)
LB_SEP_2.Name = "LB_SEP_2"
LB_SEP_2.Size = New System.Drawing.Size(709, 1)
LB_SEP_2.Size = New System.Drawing.Size(589, 1)
LB_SEP_2.TabIndex = 5
'
'TP_WHAT
@@ -519,7 +525,7 @@ Namespace API.YouTube.Controls
'
LBL_SUBS_FORMAT.AutoSize = True
LBL_SUBS_FORMAT.Dock = System.Windows.Forms.DockStyle.Fill
LBL_SUBS_FORMAT.Location = New System.Drawing.Point(552, 0)
LBL_SUBS_FORMAT.Location = New System.Drawing.Point(432, 0)
LBL_SUBS_FORMAT.Name = "LBL_SUBS_FORMAT"
LBL_SUBS_FORMAT.Size = New System.Drawing.Size(74, 28)
LBL_SUBS_FORMAT.TabIndex = 2
@@ -531,7 +537,7 @@ Namespace API.YouTube.Controls
'
Me.LBL_AUDIO_CODEC.AutoSize = True
Me.LBL_AUDIO_CODEC.Dock = System.Windows.Forms.DockStyle.Fill
Me.LBL_AUDIO_CODEC.Location = New System.Drawing.Point(552, 0)
Me.LBL_AUDIO_CODEC.Location = New System.Drawing.Point(432, 0)
Me.LBL_AUDIO_CODEC.Name = "LBL_AUDIO_CODEC"
Me.LBL_AUDIO_CODEC.Size = New System.Drawing.Size(74, 28)
Me.LBL_AUDIO_CODEC.TabIndex = 5
@@ -539,6 +545,59 @@ Namespace API.YouTube.Controls
Me.LBL_AUDIO_CODEC.TextAlign = System.Drawing.ContentAlignment.MiddleRight
TT_MAIN.SetToolTip(Me.LBL_AUDIO_CODEC, "Output Audio Codec")
'
'TP_FPS_BITRATE
'
TP_FPS_BITRATE.ColumnCount = 2
TP_FPS_BITRATE.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 50.0!))
TP_FPS_BITRATE.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 50.0!))
TP_FPS_BITRATE.Controls.Add(Me.TXT_FPS, 0, 0)
TP_FPS_BITRATE.Controls.Add(Me.TXT_AUDIO_BITRATE, 1, 0)
TP_FPS_BITRATE.Dock = System.Windows.Forms.DockStyle.Fill
TP_FPS_BITRATE.Location = New System.Drawing.Point(6, 93)
TP_FPS_BITRATE.Margin = New System.Windows.Forms.Padding(6, 0, 6, 0)
TP_FPS_BITRATE.Name = "TP_FPS_BITRATE"
TP_FPS_BITRATE.RowCount = 1
TP_FPS_BITRATE.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_FPS_BITRATE.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
TP_FPS_BITRATE.Size = New System.Drawing.Size(589, 28)
TP_FPS_BITRATE.TabIndex = 6
'
'TXT_FPS
'
ActionButton7.BackgroundImage = CType(resources.GetObject("ActionButton7.BackgroundImage"), System.Drawing.Image)
ActionButton7.Name = "Clear"
ActionButton7.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
Me.TXT_FPS.Buttons.Add(ActionButton7)
Me.TXT_FPS.CaptionText = "Video FPS"
Me.TXT_FPS.CaptionToolTipEnabled = True
Me.TXT_FPS.CaptionToolTipText = "Set the video FPS by setting the FPS value in this field. Leave blank so as not t" &
"o change"
Me.TXT_FPS.CaptionWidth = 60.0R
Me.TXT_FPS.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_FPS.Location = New System.Drawing.Point(3, 2)
Me.TXT_FPS.Margin = New System.Windows.Forms.Padding(3, 2, 3, 3)
Me.TXT_FPS.Name = "TXT_FPS"
Me.TXT_FPS.Size = New System.Drawing.Size(288, 22)
Me.TXT_FPS.TabIndex = 0
Me.TXT_FPS.TextBoxWidthMinimal = 30
'
'TXT_AUDIO_BITRATE
'
ActionButton8.BackgroundImage = CType(resources.GetObject("ActionButton8.BackgroundImage"), System.Drawing.Image)
ActionButton8.Name = "Clear"
ActionButton8.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
Me.TXT_AUDIO_BITRATE.Buttons.Add(ActionButton8)
Me.TXT_AUDIO_BITRATE.CaptionText = "Audio bitrate"
Me.TXT_AUDIO_BITRATE.CaptionToolTipEnabled = True
Me.TXT_AUDIO_BITRATE.CaptionToolTipText = "Set the video FPS if you want to change it during download. Leave blank so as not" &
" to change."
Me.TXT_AUDIO_BITRATE.CaptionWidth = 75.0R
Me.TXT_AUDIO_BITRATE.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_AUDIO_BITRATE.Location = New System.Drawing.Point(297, 3)
Me.TXT_AUDIO_BITRATE.Name = "TXT_AUDIO_BITRATE"
Me.TXT_AUDIO_BITRATE.Size = New System.Drawing.Size(289, 22)
Me.TXT_AUDIO_BITRATE.TabIndex = 1
'
'TP_HEADER_BASE
'
Me.TP_HEADER_BASE.CellBorderStyle = System.Windows.Forms.TableLayoutPanelCellBorderStyle.[Single]
@@ -553,8 +612,8 @@ Namespace API.YouTube.Controls
Me.TP_HEADER_BASE.RowCount = 1
Me.TP_HEADER_BASE.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.TP_HEADER_BASE.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 64.0!))
Me.TP_HEADER_BASE.Size = New System.Drawing.Size(721, 65)
Me.TP_HEADER_BASE.TabIndex = 6
Me.TP_HEADER_BASE.Size = New System.Drawing.Size(601, 65)
Me.TP_HEADER_BASE.TabIndex = 7
'
'TP_SUBS
'
@@ -566,31 +625,31 @@ Namespace API.YouTube.Controls
Me.TP_SUBS.Controls.Add(LBL_SUBS_FORMAT, 1, 0)
Me.TP_SUBS.Controls.Add(Me.CMB_SUBS_FORMAT, 2, 0)
Me.TP_SUBS.Dock = System.Windows.Forms.DockStyle.Fill
Me.TP_SUBS.Location = New System.Drawing.Point(6, 93)
Me.TP_SUBS.Location = New System.Drawing.Point(6, 121)
Me.TP_SUBS.Margin = New System.Windows.Forms.Padding(6, 0, 6, 0)
Me.TP_SUBS.Name = "TP_SUBS"
Me.TP_SUBS.RowCount = 1
Me.TP_SUBS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.TP_SUBS.Size = New System.Drawing.Size(709, 28)
Me.TP_SUBS.Size = New System.Drawing.Size(589, 28)
Me.TP_SUBS.TabIndex = 2
'
'TXT_SUBS
'
ActionButton7.BackgroundImage = CType(resources.GetObject("ActionButton7.BackgroundImage"), System.Drawing.Image)
ActionButton7.Name = "Open"
ActionButton7.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open
ActionButton7.ToolTipText = "Choose subtitles"
ActionButton8.BackgroundImage = CType(resources.GetObject("ActionButton8.BackgroundImage"), System.Drawing.Image)
ActionButton8.Name = "Refresh"
ActionButton8.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh
ActionButton8.ToolTipText = "Reset subtitles to initial selected"
ActionButton9.BackgroundImage = CType(resources.GetObject("ActionButton9.BackgroundImage"), System.Drawing.Image)
ActionButton9.Name = "Clear"
ActionButton9.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
ActionButton9.ToolTipText = "Clear subtitles selection (don't download subtitles)"
Me.TXT_SUBS.Buttons.Add(ActionButton7)
Me.TXT_SUBS.Buttons.Add(ActionButton8)
ActionButton9.Name = "Open"
ActionButton9.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open
ActionButton9.ToolTipText = "Choose subtitles"
ActionButton10.BackgroundImage = CType(resources.GetObject("ActionButton10.BackgroundImage"), System.Drawing.Image)
ActionButton10.Name = "Refresh"
ActionButton10.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh
ActionButton10.ToolTipText = "Reset subtitles to initial selected"
ActionButton11.BackgroundImage = CType(resources.GetObject("ActionButton11.BackgroundImage"), System.Drawing.Image)
ActionButton11.Name = "Clear"
ActionButton11.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
ActionButton11.ToolTipText = "Clear subtitles selection (don't download subtitles)"
Me.TXT_SUBS.Buttons.Add(ActionButton9)
Me.TXT_SUBS.Buttons.Add(ActionButton10)
Me.TXT_SUBS.Buttons.Add(ActionButton11)
Me.TXT_SUBS.CaptionText = "Subtitles"
Me.TXT_SUBS.CaptionToolTipEnabled = True
Me.TXT_SUBS.CaptionToolTipText = "The selected subtitles will also be downloaded"
@@ -599,7 +658,7 @@ Namespace API.YouTube.Controls
Me.TXT_SUBS.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_SUBS.Location = New System.Drawing.Point(3, 3)
Me.TXT_SUBS.Name = "TXT_SUBS"
Me.TXT_SUBS.Size = New System.Drawing.Size(543, 22)
Me.TXT_SUBS.Size = New System.Drawing.Size(423, 22)
Me.TXT_SUBS.TabIndex = 0
Me.TXT_SUBS.TextBoxReadOnly = True
'
@@ -608,7 +667,7 @@ Namespace API.YouTube.Controls
Me.CMB_SUBS_FORMAT.Dock = System.Windows.Forms.DockStyle.Fill
Me.CMB_SUBS_FORMAT.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList
Me.CMB_SUBS_FORMAT.FormattingEnabled = True
Me.CMB_SUBS_FORMAT.Location = New System.Drawing.Point(632, 3)
Me.CMB_SUBS_FORMAT.Location = New System.Drawing.Point(512, 3)
Me.CMB_SUBS_FORMAT.Name = "CMB_SUBS_FORMAT"
Me.CMB_SUBS_FORMAT.Size = New System.Drawing.Size(74, 21)
Me.CMB_SUBS_FORMAT.TabIndex = 1
@@ -618,55 +677,56 @@ Namespace API.YouTube.Controls
Me.TP_MAIN.ColumnCount = 1
Me.TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.TP_MAIN.Controls.Add(Me.TP_HEADER_BASE, 0, 0)
Me.TP_MAIN.Controls.Add(TP_FOOTER, 0, 8)
Me.TP_MAIN.Controls.Add(TP_FOOTER, 0, 9)
Me.TP_MAIN.Controls.Add(Me.TP_OPTIONS, 0, 1)
Me.TP_MAIN.Controls.Add(Me.TP_CONTROLS, 0, 6)
Me.TP_MAIN.Controls.Add(LB_SEP_1, 0, 5)
Me.TP_MAIN.Controls.Add(LB_SEP_2, 0, 7)
Me.TP_MAIN.Controls.Add(Me.TP_SUBS, 0, 2)
Me.TP_MAIN.Controls.Add(Me.TXT_SUBS_ADDIT, 0, 3)
Me.TP_MAIN.Controls.Add(Me.TXT_EXTRA_AUDIO_FORMATS, 0, 4)
Me.TP_MAIN.Controls.Add(Me.TP_CONTROLS, 0, 7)
Me.TP_MAIN.Controls.Add(LB_SEP_1, 0, 6)
Me.TP_MAIN.Controls.Add(LB_SEP_2, 0, 8)
Me.TP_MAIN.Controls.Add(Me.TP_SUBS, 0, 3)
Me.TP_MAIN.Controls.Add(Me.TXT_SUBS_ADDIT, 0, 4)
Me.TP_MAIN.Controls.Add(Me.TXT_EXTRA_AUDIO_FORMATS, 0, 5)
Me.TP_MAIN.Controls.Add(TP_FPS_BITRATE, 0, 2)
Me.TP_MAIN.Dock = System.Windows.Forms.DockStyle.Fill
Me.TP_MAIN.Location = New System.Drawing.Point(0, 0)
Me.TP_MAIN.Name = "TP_MAIN"
Me.TP_MAIN.RowCount = 10
Me.TP_MAIN.RowCount = 11
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 65.0!))
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 5.0!))
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 5.0!))
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 87.0!))
Me.TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle())
Me.TP_MAIN.Size = New System.Drawing.Size(721, 300)
Me.TP_MAIN.Size = New System.Drawing.Size(601, 328)
Me.TP_MAIN.TabIndex = 0
'
'TP_OPTIONS
'
Me.TP_OPTIONS.ColumnCount = 7
Me.TP_OPTIONS.ColumnCount = 6
Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 80.0!))
Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 80.0!))
Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 80.0!))
Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 120.0!))
Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 80.0!))
Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 80.0!))
Me.TP_OPTIONS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 20.0!))
Me.TP_OPTIONS.Controls.Add(LBL_FORMAT, 1, 0)
Me.TP_OPTIONS.Controls.Add(TP_WHAT, 0, 0)
Me.TP_OPTIONS.Controls.Add(Me.CMB_FORMAT, 2, 0)
Me.TP_OPTIONS.Controls.Add(Me.LBL_AUDIO_CODEC, 5, 0)
Me.TP_OPTIONS.Controls.Add(Me.CMB_AUDIO_CODEC, 6, 0)
Me.TP_OPTIONS.Controls.Add(Me.LBL_AUDIO_CODEC, 4, 0)
Me.TP_OPTIONS.Controls.Add(Me.CMB_AUDIO_CODEC, 5, 0)
Me.TP_OPTIONS.Controls.Add(Me.NUM_RES, 3, 0)
Me.TP_OPTIONS.Controls.Add(Me.TXT_FPS, 4, 0)
Me.TP_OPTIONS.Dock = System.Windows.Forms.DockStyle.Fill
Me.TP_OPTIONS.Location = New System.Drawing.Point(6, 65)
Me.TP_OPTIONS.Margin = New System.Windows.Forms.Padding(6, 0, 6, 0)
Me.TP_OPTIONS.Name = "TP_OPTIONS"
Me.TP_OPTIONS.RowCount = 1
Me.TP_OPTIONS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.TP_OPTIONS.Size = New System.Drawing.Size(709, 28)
Me.TP_OPTIONS.Size = New System.Drawing.Size(589, 28)
Me.TP_OPTIONS.TabIndex = 1
'
'CMB_FORMAT
@@ -684,7 +744,7 @@ Namespace API.YouTube.Controls
Me.CMB_AUDIO_CODEC.Dock = System.Windows.Forms.DockStyle.Fill
Me.CMB_AUDIO_CODEC.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList
Me.CMB_AUDIO_CODEC.FormattingEnabled = True
Me.CMB_AUDIO_CODEC.Location = New System.Drawing.Point(632, 3)
Me.CMB_AUDIO_CODEC.Location = New System.Drawing.Point(512, 3)
Me.CMB_AUDIO_CODEC.Name = "CMB_AUDIO_CODEC"
Me.CMB_AUDIO_CODEC.Size = New System.Drawing.Size(74, 21)
Me.CMB_AUDIO_CODEC.TabIndex = 3
@@ -701,57 +761,39 @@ Namespace API.YouTube.Controls
Me.NUM_RES.TextAlign = System.Windows.Forms.HorizontalAlignment.Center
Me.NUM_RES.Value = New Decimal(New Integer() {1080, 0, 0, 0})
'
'TXT_FPS
'
ActionButton10.BackgroundImage = CType(resources.GetObject("ActionButton10.BackgroundImage"), System.Drawing.Image)
ActionButton10.Name = "Clear"
ActionButton10.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
Me.TXT_FPS.Buttons.Add(ActionButton10)
Me.TXT_FPS.CaptionText = "FPS"
Me.TXT_FPS.CaptionToolTipEnabled = True
Me.TXT_FPS.CaptionToolTipText = "You can reduce the video FPS by setting the FPS value in this field."
Me.TXT_FPS.CaptionWidth = 30.0R
Me.TXT_FPS.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_FPS.Location = New System.Drawing.Point(432, 2)
Me.TXT_FPS.Margin = New System.Windows.Forms.Padding(3, 2, 3, 3)
Me.TXT_FPS.Name = "TXT_FPS"
Me.TXT_FPS.Size = New System.Drawing.Size(114, 22)
Me.TXT_FPS.TabIndex = 6
Me.TXT_FPS.TextBoxWidthMinimal = 30
'
'TP_CONTROLS
'
Me.TP_CONTROLS.ColumnCount = 1
Me.TP_CONTROLS.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.TP_CONTROLS.Dock = System.Windows.Forms.DockStyle.Fill
Me.TP_CONTROLS.Location = New System.Drawing.Point(3, 182)
Me.TP_CONTROLS.Location = New System.Drawing.Point(3, 210)
Me.TP_CONTROLS.Margin = New System.Windows.Forms.Padding(3, 0, 3, 0)
Me.TP_CONTROLS.Name = "TP_CONTROLS"
Me.TP_CONTROLS.RowCount = 1
Me.TP_CONTROLS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.TP_CONTROLS.Size = New System.Drawing.Size(715, 25)
Me.TP_CONTROLS.Size = New System.Drawing.Size(595, 25)
Me.TP_CONTROLS.TabIndex = 0
'
'TXT_SUBS_ADDIT
'
ActionButton11.BackgroundImage = CType(resources.GetObject("ActionButton11.BackgroundImage"), System.Drawing.Image)
ActionButton11.Enabled = False
ActionButton11.Name = "Open"
ActionButton11.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open
ActionButton11.ToolTipText = "Choose additional formats"
ActionButton12.BackgroundImage = CType(resources.GetObject("ActionButton12.BackgroundImage"), System.Drawing.Image)
ActionButton12.Enabled = False
ActionButton12.Name = "Refresh"
ActionButton12.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh
ActionButton12.ToolTipText = "Fill in additional formats from the defaults"
ActionButton12.Name = "Open"
ActionButton12.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open
ActionButton12.ToolTipText = "Choose additional formats"
ActionButton13.BackgroundImage = CType(resources.GetObject("ActionButton13.BackgroundImage"), System.Drawing.Image)
ActionButton13.Enabled = False
ActionButton13.Name = "Clear"
ActionButton13.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
ActionButton13.ToolTipText = "Remove all additional formats"
Me.TXT_SUBS_ADDIT.Buttons.Add(ActionButton11)
ActionButton13.Name = "Refresh"
ActionButton13.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh
ActionButton13.ToolTipText = "Fill in additional formats from the defaults"
ActionButton14.BackgroundImage = CType(resources.GetObject("ActionButton14.BackgroundImage"), System.Drawing.Image)
ActionButton14.Enabled = False
ActionButton14.Name = "Clear"
ActionButton14.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
ActionButton14.ToolTipText = "Remove all additional formats"
Me.TXT_SUBS_ADDIT.Buttons.Add(ActionButton12)
Me.TXT_SUBS_ADDIT.Buttons.Add(ActionButton13)
Me.TXT_SUBS_ADDIT.Buttons.Add(ActionButton14)
Me.TXT_SUBS_ADDIT.CaptionMode = PersonalUtilities.Forms.Controls.Base.ICaptionControl.Modes.CheckBox
Me.TXT_SUBS_ADDIT.CaptionText = "Additional subtitle formats"
Me.TXT_SUBS_ADDIT.CaptionToolTipEnabled = True
@@ -759,44 +801,44 @@ Namespace API.YouTube.Controls
Me.TXT_SUBS_ADDIT.CaptionWidth = 150.0R
Me.TXT_SUBS_ADDIT.ClearTextByButtonClear = False
Me.TXT_SUBS_ADDIT.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_SUBS_ADDIT.Location = New System.Drawing.Point(6, 124)
Me.TXT_SUBS_ADDIT.Location = New System.Drawing.Point(6, 152)
Me.TXT_SUBS_ADDIT.Margin = New System.Windows.Forms.Padding(6, 3, 6, 3)
Me.TXT_SUBS_ADDIT.Name = "TXT_SUBS_ADDIT"
Me.TXT_SUBS_ADDIT.Size = New System.Drawing.Size(709, 22)
Me.TXT_SUBS_ADDIT.Size = New System.Drawing.Size(589, 22)
Me.TXT_SUBS_ADDIT.TabIndex = 3
Me.TXT_SUBS_ADDIT.Tag = "s"
Me.TXT_SUBS_ADDIT.TextBoxReadOnly = True
'
'TXT_EXTRA_AUDIO_FORMATS
'
ActionButton14.BackgroundImage = CType(resources.GetObject("ActionButton14.BackgroundImage"), System.Drawing.Image)
ActionButton14.Enabled = False
ActionButton14.Name = "Open"
ActionButton14.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open
ActionButton14.ToolTipText = "Choose additional formats"
ActionButton15.BackgroundImage = CType(resources.GetObject("ActionButton15.BackgroundImage"), System.Drawing.Image)
ActionButton15.Enabled = False
ActionButton15.Name = "Refresh"
ActionButton15.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh
ActionButton15.ToolTipText = "Fill in additional formats from the defaults"
ActionButton15.Name = "Open"
ActionButton15.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Open
ActionButton15.ToolTipText = "Choose additional formats"
ActionButton16.BackgroundImage = CType(resources.GetObject("ActionButton16.BackgroundImage"), System.Drawing.Image)
ActionButton16.Enabled = False
ActionButton16.Name = "Clear"
ActionButton16.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
ActionButton16.ToolTipText = "Choose additional formats"
Me.TXT_EXTRA_AUDIO_FORMATS.Buttons.Add(ActionButton14)
ActionButton16.Name = "Refresh"
ActionButton16.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Refresh
ActionButton16.ToolTipText = "Fill in additional formats from the defaults"
ActionButton17.BackgroundImage = CType(resources.GetObject("ActionButton17.BackgroundImage"), System.Drawing.Image)
ActionButton17.Enabled = False
ActionButton17.Name = "Clear"
ActionButton17.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
ActionButton17.ToolTipText = "Choose additional formats"
Me.TXT_EXTRA_AUDIO_FORMATS.Buttons.Add(ActionButton15)
Me.TXT_EXTRA_AUDIO_FORMATS.Buttons.Add(ActionButton16)
Me.TXT_EXTRA_AUDIO_FORMATS.Buttons.Add(ActionButton17)
Me.TXT_EXTRA_AUDIO_FORMATS.CaptionMode = PersonalUtilities.Forms.Controls.Base.ICaptionControl.Modes.CheckBox
Me.TXT_EXTRA_AUDIO_FORMATS.CaptionText = "Additional audio formats"
Me.TXT_EXTRA_AUDIO_FORMATS.CaptionToolTipEnabled = True
Me.TXT_EXTRA_AUDIO_FORMATS.CaptionWidth = 150.0R
Me.TXT_EXTRA_AUDIO_FORMATS.ClearTextByButtonClear = False
Me.TXT_EXTRA_AUDIO_FORMATS.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_EXTRA_AUDIO_FORMATS.Location = New System.Drawing.Point(6, 152)
Me.TXT_EXTRA_AUDIO_FORMATS.Location = New System.Drawing.Point(6, 180)
Me.TXT_EXTRA_AUDIO_FORMATS.Margin = New System.Windows.Forms.Padding(6, 3, 6, 3)
Me.TXT_EXTRA_AUDIO_FORMATS.Name = "TXT_EXTRA_AUDIO_FORMATS"
Me.TXT_EXTRA_AUDIO_FORMATS.Size = New System.Drawing.Size(709, 22)
Me.TXT_EXTRA_AUDIO_FORMATS.Size = New System.Drawing.Size(589, 22)
Me.TXT_EXTRA_AUDIO_FORMATS.TabIndex = 4
Me.TXT_EXTRA_AUDIO_FORMATS.Tag = "a"
Me.TXT_EXTRA_AUDIO_FORMATS.TextBoxReadOnly = True
@@ -807,14 +849,14 @@ Namespace API.YouTube.Controls
Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)
Me.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font
Me.CancelButton = Me.BTT_CANCEL
Me.ClientSize = New System.Drawing.Size(721, 300)
Me.ClientSize = New System.Drawing.Size(601, 328)
Me.Controls.Add(Me.TP_MAIN)
Me.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedSingle
Me.Icon = Global.SCrawler.My.Resources.SiteYouTube.YouTubeIcon_32
Me.KeyPreview = True
Me.MaximizeBox = False
Me.MinimizeBox = False
Me.MinimumSize = New System.Drawing.Size(737, 339)
Me.MinimumSize = New System.Drawing.Size(617, 367)
Me.Name = "VideoOptionsForm"
Me.ShowInTaskbar = False
Me.SizeGripStyle = System.Windows.Forms.SizeGripStyle.Hide
@@ -834,6 +876,9 @@ Namespace API.YouTube.Controls
CType(Me.CMB_PLS, System.ComponentModel.ISupportInitialize).EndInit()
TP_WHAT.ResumeLayout(False)
TP_WHAT.PerformLayout()
TP_FPS_BITRATE.ResumeLayout(False)
CType(Me.TXT_FPS, System.ComponentModel.ISupportInitialize).EndInit()
CType(Me.TXT_AUDIO_BITRATE, System.ComponentModel.ISupportInitialize).EndInit()
Me.TP_HEADER_BASE.ResumeLayout(False)
Me.TP_SUBS.ResumeLayout(False)
Me.TP_SUBS.PerformLayout()
@@ -842,7 +887,6 @@ Namespace API.YouTube.Controls
Me.TP_OPTIONS.ResumeLayout(False)
Me.TP_OPTIONS.PerformLayout()
CType(Me.NUM_RES, System.ComponentModel.ISupportInitialize).EndInit()
CType(Me.TXT_FPS, System.ComponentModel.ISupportInitialize).EndInit()
CType(Me.TXT_SUBS_ADDIT, System.ComponentModel.ISupportInitialize).EndInit()
CType(Me.TXT_EXTRA_AUDIO_FORMATS, System.ComponentModel.ISupportInitialize).EndInit()
Me.ResumeLayout(False)
@@ -875,5 +919,6 @@ Namespace API.YouTube.Controls
Private WithEvents TXT_FPS As PersonalUtilities.Forms.Controls.TextBoxExtended
Private WithEvents CMB_PLS As PersonalUtilities.Forms.Controls.ComboBoxExtended
Private WithEvents BTT_PLS_BROWSE As Button
Private WithEvents TXT_AUDIO_BITRATE As PersonalUtilities.Forms.Controls.TextBoxExtended
End Class
End Namespace

71
SCrawler.YouTube/Controls/VideoOptionsForm.resx
View File

@@ -377,50 +377,61 @@
<metadata name="LBL_SUBS_FORMAT.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<value>False</value>
</metadata>
<metadata name="TP_FPS_BITRATE.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<value>False</value>
</metadata>
<data name="ActionButton7.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
wwAADsMBx2+oZAAAAR5JREFUOE+VkjFqwzAUhn2D9iShRyi+QhYbGujg3ZATZPKYdC6FQhPwlAMkg3dP
WQwhyWIyJIUW5NqyPb7oCVtIlhVTwYf8nv7/t2zJagel9KmqKsIACYL9RjI8UHz5zshougZr/AEvbxEP
aZCDBY3VslixaJvX3wzkkDiOwbZtDRGA5vdNAg+TL27qgmt5XkBG/gTdAG7Gt+3PP9oOaEGFCVEC6rp+
5g9MfM/c5e4OsEZMZkQEtGL5H2DdZ5JRArDwPA+iKII0TfkC9vroC9j5vq8JTWw3WzWgLMtZGIaa0MR8
vlAD8PYlSaIJTTiOowY0p0Bc19XEJo6HE59FAPuMzyAINKGJ1XLFZxHALtMrnkBXOIQIIIQ8YvF/KrgB
cMaRN0UdBBkAAAAASUVORK5CYII=
xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go
tbbB43rK5xSAQq1VYFtmeQBoqZTSreVZvgTknM8yyyjA/qodsDF9gspD2Bj6B+DH+NqzhQQAG+POMnSX
AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC
</value>
</data>
<data name="ActionButton8.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6
JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE
QVQ4T2P4//8/QczOJyyqHpzfiE0OQwAZC8iqszAzs7CJ69o4BR768V/W2jcGXQ0KB4aFNS3dDQtnrbCb
ePCK48wTN1wXXXzge/jXf/clV55zC4hIIatF0cjIyMikElzc57z0wX+XHd/+2+//99/ywP//xlu//tdb
+eK/4Zp3/1WTOhYzARViNUAluKjTdf37/0ZTTn9TbdhwXblhwwW1/qOP1Ja9+K8w+95/6cm3/6v2Xvkv
qKjniGGAoIqRpW3/4e8S9uGdzFz82gwMDFxAzCxm4ZegtuLDf+VJ1/8rZM25IqLvnM/CximCYYCic1QN
v7x2JIwPwyrJ3XNUylddE9G2TWNmZOBDl4czmJiZMSRBmFdSyYyJgUEQmxwIYxWEYXZBCUls4sgYq6CA
prWNbtG8nXKeaVPR5XiVjSxEzf0yYXy4BBMLO6eQjoOXZvrkbbazrv53Xf/2v4CSbjBMXkhBl1/CMyNZ
qWnvGy5pNQ+YONwAfjXzAOupl/47LLr333L50/96q9/8l23YdES6cO5KuYqVW+R7Tj6SnfP0v4hryjyY
HhQDmFjYeHVKFp7WX/Xuv9Kq9/+Vd/z7r7rv/3+l7f//y676DEwDN/9L+BVvYkKLCTgDhNkkVUyVlr74
qbbz73/VOTc/qsy89kWx+9h7qbQpJwS1bbOAscGGrB6EUTggLOqf16C55ft/HlnNAFZOXgVWdi4FRgYG
VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg==
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go
tbbB43rK5xSAQq1VYFtmeQBoqZTSreVZvgTknM8yyyjA/qodsDF9gspD2Bj6B+DH+NqzhQQAG+POMnSX
AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC
</value>
</data>
<data name="ActionButton9.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go
tbbB43rK5xSAQq1VYFtmeQBoqZTSreVZvgTknM8yyyjA/qodsDF9gspD2Bj6B+DH+NqzhQQAG+POMnSX
AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC
wwAADsMBx2+oZAAAAR5JREFUOE+VkjFqwzAUhn2D9iShRyi+QhYbGujg3ZATZPKYdC6FQhPwlAMkg3dP
WQwhyWIyJIUW5NqyPb7oCVtIlhVTwYf8nv7/t2zJagel9KmqKsIACYL9RjI8UHz5zshougZr/AEvbxEP
aZCDBY3VslixaJvX3wzkkDiOwbZtDRGA5vdNAg+TL27qgmt5XkBG/gTdAG7Gt+3PP9oOaEGFCVEC6rp+
5g9MfM/c5e4OsEZMZkQEtGL5H2DdZ5JRArDwPA+iKII0TfkC9vroC9j5vq8JTWw3WzWgLMtZGIaa0MR8
vlAD8PYlSaIJTTiOowY0p0Bc19XEJo6HE59FAPuMzyAINKGJ1XLFZxHALtMrnkBXOIQIIIQ8YvF/KrgB
cMaRN0UdBBkAAAAASUVORK5CYII=
</value>
</data>
<data name="ActionButton10.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go
tbbB43rK5xSAQq1VYFtmeQBoqZTSreVZvgTknM8yyyjA/qodsDF9gspD2Bj6B+DH+NqzhQQAG+POMnSX
AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6
JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE
QVQ4T2P4//8/QczOJyyqHpzfiE0OQwAZC8iqszAzs7CJ69o4BR768V/W2jcGXQ0KB4aFNS3dDQtnrbCb
ePCK48wTN1wXXXzge/jXf/clV55zC4hIIatF0cjIyMikElzc57z0wX+XHd/+2+//99/ywP//xlu//tdb
+eK/4Zp3/1WTOhYzARViNUAluKjTdf37/0ZTTn9TbdhwXblhwwW1/qOP1Ja9+K8w+95/6cm3/6v2Xvkv
qKjniGGAoIqRpW3/4e8S9uGdzFz82gwMDFxAzCxm4ZegtuLDf+VJ1/8rZM25IqLvnM/CximCYYCic1QN
v7x2JIwPwyrJ3XNUylddE9G2TWNmZOBDl4czmJiZMSRBmFdSyYyJgUEQmxwIYxWEYXZBCUls4sgYq6CA
prWNbtG8nXKeaVPR5XiVjSxEzf0yYXy4BBMLO6eQjoOXZvrkbbazrv53Xf/2v4CSbjBMXkhBl1/CMyNZ
qWnvGy5pNQ+YONwAfjXzAOupl/47LLr333L50/96q9/8l23YdES6cO5KuYqVW+R7Tj6SnfP0v4hryjyY
HhQDmFjYeHVKFp7WX/Xuv9Kq9/+Vd/z7r7rv/3+l7f//y676DEwDN/9L+BVvYkKLCTgDhNkkVUyVlr74
qbbz73/VOTc/qsy89kWx+9h7qbQpJwS1bbOAscGGrB6EUTggLOqf16C55ft/HlnNAFZOXgVWdi4FRgYG
VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg==
</value>
</data>
<data name="ActionButton11.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go
tbbB43rK5xSAQq1VYFtmeQBoqZTSreVZvgTknM8yyyjA/qodsDF9gspD2Bj6B+DH+NqzhQQAG+POMnSX
AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC
</value>
</data>
<data name="ActionButton12.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
wwAADsMBx2+oZAAAAR5JREFUOE+VkjFqwzAUhn2D9iShRyi+QhYbGujg3ZATZPKYdC6FQhPwlAMkg3dP
@@ -431,7 +442,7 @@
cMaRN0UdBBkAAAAASUVORK5CYII=
</value>
</data>
<data name="ActionButton12.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<data name="ActionButton13.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6
JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE
@@ -447,7 +458,7 @@
VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg==
</value>
</data>
<data name="ActionButton13.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<data name="ActionButton14.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go
@@ -455,7 +466,7 @@
AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC
</value>
</data>
<data name="ActionButton14.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<data name="ActionButton15.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
wwAADsMBx2+oZAAAAR5JREFUOE+VkjFqwzAUhn2D9iShRyi+QhYbGujg3ZATZPKYdC6FQhPwlAMkg3dP
@@ -466,7 +477,7 @@
cMaRN0UdBBkAAAAASUVORK5CYII=
</value>
</data>
<data name="ActionButton15.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<data name="ActionButton16.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6
JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE
@@ -482,7 +493,7 @@
VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg==
</value>
</data>
<data name="ActionButton16.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<data name="ActionButton17.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go

27
SCrawler.YouTube/Controls/VideoOptionsForm.vb
View File

@@ -26,7 +26,7 @@ Namespace API.YouTube.Controls
Friend Property DesignXML As EContainer Implements IDesignXMLContainer.DesignXML
Private Property DesignXMLNodes As String() Implements IDesignXMLContainer.DesignXMLNodes
Private Property DesignXMLNodeName As String Implements IDesignXMLContainer.DesignXMLNodeName
Private Const ControlsRow As Integer = 6
Private Const ControlsRow As Integer = 7
Private ReadOnly Property CNT_PROCESSOR As TableControlsProcessor
Friend Property MyContainer As YouTubeMediaContainerBase
Private Initialization As Boolean = True
@@ -164,11 +164,16 @@ Namespace API.YouTube.Controls
If InheritsFromContainer Then
If .OutputVideoFPS > 0 Then TXT_FPS.Text = .OutputVideoFPS
If .OutputAudioBitrate > 0 Then TXT_AUDIO_BITRATE.Text = .OutputAudioBitrate
Else
If MyYouTubeSettings.DefaultVideoFPS > 0 Then TXT_FPS.Text = MyYouTubeSettings.DefaultVideoFPS
If MyYouTubeSettings.DefaultAudioBitrate > 0 Then TXT_AUDIO_BITRATE.Text = MyYouTubeSettings.DefaultAudioBitrate.Value
End If
MyFieldsChecker.AddControl(Of Double)(TXT_FPS, TXT_FPS.CaptionText, True, New FpsFieldChecker)
MyFieldsChecker.EndLoaderOperations()
With MyFieldsChecker
.AddControl(Of Double)(TXT_FPS, TXT_FPS.CaptionText, True, New FpsFieldChecker)
.AddControl(Of Integer)(TXT_AUDIO_BITRATE, TXT_AUDIO_BITRATE.CaptionText, True)
.EndLoaderOperations()
End With
TP_SUBS.Enabled = .Subtitles.Count > 0
TXT_SUBS_ADDIT.Enabled = .Subtitles.Count > 0
RefillTextBoxes()
@@ -327,6 +332,7 @@ Namespace API.YouTube.Controls
If Full Then
.OutputVideoExtension = CMB_FORMAT.Text.StringToLower
.OutputVideoFPS = AConvert(Of Double)(TXT_FPS.Text, YouTubeSettings.FpsFormatProvider.MyProviderDefault, -1)
.OutputAudioBitrate = AConvert(Of Integer)(TXT_AUDIO_BITRATE.Text, -1)
.OutputAudioCodec = CMB_AUDIO_CODEC.Text.StringToLower
.OutputSubtitlesFormat = CMB_SUBS_FORMAT.Text.StringToLower
.IsAudioSelected = OPT_AUDIO.Checked
@@ -346,10 +352,12 @@ Namespace API.YouTube.Controls
Else
f = TXT_FILE.Text
End If
f = CleanFileName(f)
If f.IsEmptyString Then Throw New ArgumentNullException("File", "The output file cannot be null")
With MyContainer
.OutputVideoExtension = CMB_FORMAT.Text.StringToLower
.OutputVideoFPS = AConvert(Of Double)(TXT_FPS.Text, YouTubeSettings.FpsFormatProvider.MyProviderDefault, -1)
.OutputAudioBitrate = AConvert(Of Integer)(TXT_AUDIO_BITRATE.Text, -1)
.OutputAudioCodec = CMB_AUDIO_CODEC.Text.StringToLower
.OutputSubtitlesFormat = CMB_SUBS_FORMAT.Text.StringToLower
.M3U8_PlaylistFiles = M3U8FilesFull
@@ -369,6 +377,7 @@ Namespace API.YouTube.Controls
Else
.SelectedVideoIndex = -1
.SelectedAudioIndex = cntIndex
.MediaType = UMTypes.Audio
End If
.FileSetManually = True
.File = f
@@ -379,6 +388,7 @@ Namespace API.YouTube.Controls
Else
If OPT_AUDIO.Checked Then
.SetMaxResolution(-2)
.MediaType = UMTypes.Audio
Else
.SetMaxResolution(NUM_RES.Value)
End If
@@ -595,12 +605,15 @@ Namespace API.YouTube.Controls
f = SFile.SelectPath(f, "Select the destination of the video files", EDP.ReturnValue)
Else
f = TXT_FILE.Text
Dim sPattern$ = $"Video|{AvailableVideoFormats.Select(Function(vf) $"*.{vf.ToLower}").ListToString(";")}" &
$"|Audio|{AvailableAudioFormats.Select(Function(af) $"*.{af.ToLower}").ListToString(";")}" &
"|All Files|*.*"
f = SFile.SaveAs(f, "Select the destination of the video file",,, sPattern, EDP.ReturnValue)
Dim ext$ = f.Extension
Dim sPattern$ = "All Files|*.*|" &
$"Video|{AvailableVideoFormats.Select(Function(vf) $"*.{vf.ToLower}").ListToString(";")}" &
$"|Audio|{AvailableAudioFormats.Select(Function(af) $"*.{af.ToLower}").ListToString(";")}"
f = SFile.SaveAs(f, "Select the destination of the video file",, ext, sPattern, EDP.ReturnValue)
f.Extension = ext
End If
#Enable Warning
f = CleanFileName(f)
If Not f.IsEmptyString Then
If e.Button = MouseButtons.Right Then
MyYouTubeSettings.DownloadLocations.Add(f, MyDownloaderSettings.OutputPathAskForName)

22
SCrawler.YouTube/Declarations.vb
View File

@@ -17,10 +17,21 @@ Namespace API.YouTube
Public Const DownloaderDataFolderYouTube As String = DownloadObjects.STDownloader.DownloaderDataFolder & "YouTube\"
Friend Const YouTubeDownloadPathDefault As String = "YouTubeDownloads\"
Friend Const SimpleArraysFormNode As String = "SimpleFormatsChooserForm"
Private Const YTDLP_DefaultName As String = "yt-dlp"
Public Property MyYouTubeSettings As Base.YouTubeSettings
Public Property MyCache As CacheKeeper
Friend ReadOnly Property MyCacheSettings As New CacheKeeper(DownloaderDataFolderYouTube) With {.DeleteCacheOnDispose = False, .DeleteRootOnDispose = False}
Public ReadOnly Property YouTubeCookieNetscapeFile As New SFile($"Settings\Responser_{YouTubeSite}_Cookies_Netscape.txt")
Friend ReadOnly Property YTDLP_NAME As String
Get
Dim n$ = MyYouTubeSettings.YTDLP.Value.Name
If Not n.IsEmptyString Then
Return If(n.ToLower = YTDLP_DefaultName, n, $"""{n}""")
Else
Return YTDLP_DefaultName
End If
End Get
End Property
Friend ReadOnly Property AvailableSubtitlesFormats As String()
Get
Return {"ASS", "LRC", "SRT", "VTT"}
@@ -45,6 +56,17 @@ Namespace API.YouTube
Friend ReadOnly TitleHtmlConverter As Func(Of String, String) = Function(Input) Input.StringRemoveWinForbiddenSymbols().StringTrim()
Friend ReadOnly ProgressProvider As IMyProgressNumberProvider = MyProgressNumberProvider.Percentage
Public ReadOnly TrueUrlRegEx As RParams = RParams.DM(Base.YouTubeFunctions.TrueUrlPattern, 0, EDP.ReturnValue)
Friend Function CleanFileName(ByVal f As SFile) As SFile
If Not f.IsEmptyString And Not f.Name.IsEmptyString Then
Dim ff As SFile = f
ff.Name = ff.Name.StringRemoveWinForbiddenSymbols
If Not ff.Name.IsEmptyString Then ff.Name = ff.Name.Replace("%", String.Empty)
If ff.Name.IsEmptyString Then ff.Name = "file"
Return ff
Else
Return f
End If
End Function
Private Class TimeToStringConverter : Implements ICustomProvider
Private ReadOnly _Provider As New ADateTime("mm\:ss") With {.TimeParseMode = ADateTime.TimeModes.TimeSpan}
Private ReadOnly _ProviderWithHours As New ADateTime("h\:mm\:ss") With {.TimeParseMode = ADateTime.TimeModes.TimeSpan}

33
SCrawler.YouTube/Downloader/MediaItem.vb
View File

@@ -12,6 +12,7 @@ Imports SCrawler.API.YouTube.Objects
Imports SCrawler.API.YouTube.Controls
Imports PersonalUtilities.Tools
Imports PersonalUtilities.Forms.Toolbars
Imports PersonalUtilities.Functions.Messaging
Namespace DownloadObjects.STDownloader
Public Delegate Sub MediaItemEventHandler(ByVal Sender As MediaItem, ByVal Container As IYouTubeMediaContainer)
<DefaultEvent("DoubleClick"), DesignTimeVisible(False), ToolboxItem(False)>
@@ -135,7 +136,7 @@ Namespace DownloadObjects.STDownloader
LBL_TITLE.Text = .ToString(True)
If Not .SiteKey = YouTubeSiteKey And .ContentType = Plugin.UserMediaTypes.Picture Then
LBL_INFO.Text = .File.Extension.StringToUpper
ElseIf Not .IsMusic Then
ElseIf Not .IsMusic And Not (.MediaType = Plugin.UserMediaTypes.Audio Or .MediaType = Plugin.UserMediaTypes.AudioPre) Then
If .Height > 0 Then
LBL_INFO.Text = $"{ .File.Extension.StringToUpper}{d}{ .Height}p"
Else
@@ -180,10 +181,10 @@ Namespace DownloadObjects.STDownloader
With MyContainer
If Not .SiteKey = YouTubeSiteKey And .ContentType = Plugin.UserMediaTypes.Picture Then
ICON_WHAT.Image = My.Resources.ImagePic_32
ElseIf Not .IsMusic Then
ICON_WHAT.Image = My.Resources.VideoCamera_32
Else
ElseIf .IsMusic Or .MediaType = Plugin.UserMediaTypes.Audio Or .MediaType = Plugin.UserMediaTypes.AudioPre Then
ICON_WHAT.Image = My.Resources.AudioMusic_32
Else
ICON_WHAT.Image = My.Resources.VideoCamera_32
End If
End With
End Sub, EDP.None)
@@ -229,7 +230,7 @@ Namespace DownloadObjects.STDownloader
.ColumnStyles.Clear()
.ColumnCount = 0
If ContainerHasElements Or MyContainer.MediaState = Plugin.UserMediaStates.Downloaded Then
If Not MyContainer.SiteKey = YouTubeSiteKey Then UpdateMediaIcon()
UpdateMediaIcon()
If ContainerHasElements Then
BTT_OPEN_FOLDER.Visible = False
BTT_OPEN_FILE.Visible = False
@@ -476,12 +477,28 @@ Namespace DownloadObjects.STDownloader
RaiseEvent Removal(Me, MyContainer)
End Sub
Private Sub BTT_DELETE_FILE_Click(sender As Object, e As EventArgs) Handles BTT_DELETE_FILE.Click
If MsgBoxE({$"Are you sure you want to delete the following {FileOption.ToString.ToLower}:{vbCr}" &
If(FileOption = SFO.File, MyContainer.File.ToString, MyContainer.File.PathWithSeparator),
$"Deleting a {FileOption.ToString.ToLower}"}, vbExclamation,,, {"Process", "Cancel"}) = 0 Then
Dim opt$
Dim opt2$ = String.Empty
If FileOption = SFO.File Then
opt = "file"
Else
opt = "item"
opt2 = "THE ITEM MAY CONTAIN MULTIPLE FILES" & vbCr
End If
Dim b As New List(Of MsgBoxButton) From {New MsgBoxButton("Process")}
If Not opt2.IsEmptyString Then _
b.Add(New MsgBoxButton("Show files", "Show files to delete") With {
.IsDialogResultButton = False,
.CallBack = Function(r, m, bb) MsgBoxE(New MMessage($"The following files will be deleted:{vbCr}{vbCr}{MyContainer.Files.ListToString(vbCr)}",
"Files to delete",, vbExclamation) With {.Editable = True})})
b.Add(New MsgBoxButton("Cancel"))
If MsgBoxE({$"Are you sure you want to delete the following {opt}:{vbCr}{opt2}" &
If(FileOption = SFO.File, MyContainer.File.ToString, MyContainer.ToString(True)),
$"Deleting {opt}"}, vbExclamation,,, b) = 0 Then
MyContainer.Delete(True)
RaiseEvent Removal(Me, MyContainer)
End If
b.Clear()
End Sub
#End Region
#Region "ISupportInitialize Support"

7
SCrawler.YouTube/Objects/PlayList.vb
View File

@@ -19,17 +19,18 @@ Namespace API.YouTube.Objects
Dim __title$ = $" - {Title}"
If Not s.IsEmptyString Then s = $" [{s}]"
If Not PlaylistTitle.IsEmptyString And Not ForMediaItem Then t = $"{PlaylistTitle} - "
Dim c% = {Count, ElementsNumber}.Max
If IsMusic Then
If Count <= 1 Then t &= "Single" Else t &= "Album"
If c <= 1 Then t &= "Single" Else t &= "Album"
Else
t &= "Playlist"
End If
If Not PlaylistTitle.IsEmptyString And Not ForMediaItem Then t &= $" - {PlaylistTitle}"
If PlaylistTitle = Title Then __title = String.Empty
If ForMediaItem Then
Return $"{t} ({Count}){__title}"
Return $"{t} ({c}){__title}"
Else
Return $"{t} ({Count}){__title} ({AConvert(Of String)(Duration, TimeToStringProvider)}){s}"
Return $"{t} ({c}){__title} ({AConvert(Of String)(Duration, TimeToStringProvider)}){s}"
End If
End Function
Public Overrides Function Parse(ByVal Container As EContainer, ByVal Path As SFile, ByVal IsMusic As Boolean,

18
SCrawler.YouTube/Objects/Track.vb
View File

@@ -27,6 +27,7 @@ Namespace API.YouTube.Objects
Else
_File.Extension = mp3
End If
_File = CleanFileName(_File)
End If
End Sub
Public Overrides Function ToString(ByVal ForMediaItem As Boolean) As String
@@ -46,12 +47,17 @@ Namespace API.YouTube.Objects
_ObjectType = Base.YouTubeMediaType.Single
Me.IsMusic = IsMusic
If MyBase.Parse(Container, Path, IsMusic, Token, Progress) Then
Dim f As SFile = MyYouTubeSettings.OutputPath
If f.IsEmptyString Then f = "YouTubeDownloads\OutputFile.mp3"
Dim ext$ = MyYouTubeSettings.DefaultAudioCodec.Value.StringToLower
If ext.IsEmptyString Then ext = "mp3"
f.Extension = ext
File = f
With MyYouTubeSettings
Dim f As SFile = .OutputPath
If f.IsEmptyString Then f = "YouTubeDownloads\OutputFile.mp3"
Dim ext$ = .DefaultAudioCodecMusic.Value.StringToLower.IfNullOrEmpty(.DefaultAudioCodec.Value.StringToLower)
If ext.IsEmptyString Then ext = "mp3"
f.Extension = ext
'If f.Name.IsEmptyString Then f.Name = File.Name
File = f
If _File.Extension.IsEmptyString Then _File.Extension = ext
_File = CleanFileName(_File)
End With
Return True
Else
Return False

296
SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb
View File

@@ -123,6 +123,15 @@ Namespace API.YouTube.Objects
<XMLEC> Public Property UserTitle As String Implements IYouTubeMediaContainer.UserTitle
#End Region
#Region "Playlist support"
Private _ElementsNumber As Integer = 0
<XMLEC> Protected Property ElementsNumber As Integer
Get
Return If(HasElements, Count, _ElementsNumber)
End Get
Set(ByVal _ElementsNumber As Integer)
Me._ElementsNumber = _ElementsNumber
End Set
End Property
Friend ReadOnly Property Elements As List(Of IYouTubeMediaContainer) Implements IYouTubeMediaContainer.Elements
Friend ReadOnly Property HasElements As Boolean Implements IYouTubeMediaContainer.HasElements
Get
@@ -265,6 +274,18 @@ Namespace API.YouTube.Objects
PostProcessing_OutputAudioFormats.RemoveAll(Function(s) s = -1)
End If
End Sub
<XMLEC("OutputAudioBitrate")> Protected _OutputAudioBitrate As Integer = -1
Friend Property OutputAudioBitrate As Integer
Get
Return _OutputAudioBitrate
End Get
Set(ByVal NewBitrate As Integer)
If Not [Protected] Then
_OutputAudioBitrate = NewBitrate
If HasElements Then Elements.ForEach(Sub(elem) DirectCast(elem, YouTubeMediaContainerBase).OutputAudioBitrate = NewBitrate)
End If
End Set
End Property
#End Region
#Region "Subtitles"
Protected ReadOnly _Subtitles As List(Of Subtitles)
@@ -376,10 +397,13 @@ Namespace API.YouTube.Objects
End Set
End Property
Protected _Size As Integer = 0
<XMLEC("SizeRecalculated")> Protected _SizeRecalculated As Boolean = False
<XMLEC> Public Overridable Property Size As Integer Implements IDownloadableMedia.Size
Get
If HasElements Then
Return Elements.Sum(Function(e) If(e.Checked, e.Size, 0))
ElseIf _SizeRecalculated Then
Return _Size
Else
If Checked Then
If IsMusic And SelectedAudioIndex.ValueBetween(0, MediaObjects.Count - 1) Then
@@ -559,7 +583,25 @@ Namespace API.YouTube.Objects
If ObjectType = YouTubeMediaType.Single AndAlso Not GetPlayListTitle.IsEmptyString Then _SpecialPath.StringAppend(GetPlayListTitle(), "\")
If Elements.Count > 0 Then Elements.ForEach(Sub(e) e.SpecialFolder = Path)
End Sub
<XMLEC> Protected Friend ReadOnly Property Files As List(Of SFile) Implements IYouTubeMediaContainer.Files
Private ReadOnly _Files As List(Of SFile)
<XMLEC> Protected Friend Property Files As List(Of SFile) Implements IYouTubeMediaContainer.Files
Get
If HasElements Then
Return GetFilesFiles()
Else
Return _Files
End If
End Get
Set(ByVal f As List(Of SFile))
_Files.ListAddList(f, LAP.NotContainsOnly)
End Set
End Property
Protected Overloads Sub AddFile(ByVal f As SFile)
_Files.ListAddValue(f, LAP.NotContainsOnly)
End Sub
Protected Overloads Sub AddFile(ByVal f As IEnumerable(Of SFile))
_Files.ListAddList(f, LAP.NotContainsOnly)
End Sub
<XMLEC> Protected _File As SFile
<XMLEC> Protected Friend Property FileSetManually As Boolean = False
Public Property FileIgnorePlaylist As Boolean = False
@@ -628,6 +670,14 @@ Namespace API.YouTube.Objects
If HasElements And Not IsMusic Then urls.ListAddList(Elements.SelectMany(Function(elem As YouTubeMediaContainerBase) elem.GetFiles()), LAP.NotContainsOnly)
Return urls
End Function
Private Function GetFilesFiles() As IEnumerable(Of SFile)
Dim f As New List(Of SFile)
If File.Exists Then f.Add(File)
If _Files.Count > 0 Then f.AddRange(_Files)
If ThumbnailFile.Exists Then f.Add(ThumbnailFile)
If HasElements Then f.ListAddList(Elements.SelectMany(Function(elem As YouTubeMediaContainerBase) elem.GetFilesFiles()), LAP.NotContainsOnly)
Return f
End Function
Private _M3U8_PlaylistFiles As IEnumerable(Of SFile) = Nothing
Friend Property M3U8_PlaylistFiles As IEnumerable(Of SFile)
Get
@@ -647,6 +697,7 @@ Namespace API.YouTube.Objects
Private Const aac As String = "aac"
Private Const ac3 As String = "ac3"
Protected PostProcessing_AudioAC3 As Boolean = False
Protected PostProcessing_AudioMP3 As Boolean = False
Public Overridable ReadOnly Property Command(ByVal WithCookies As Boolean) As String Implements IYouTubeMediaContainer.Command
Get
If Not File.IsEmptyString Then
@@ -682,6 +733,10 @@ Namespace API.YouTube.Objects
PostProcessing_AudioAC3 = True
formats.StringAppend($"--audio-format {aac}", " ")
atCodec = aac
ElseIf SelectedVideoIndex >= 0 And OutputAudioCodec.StringToLower = mp3 Then
PostProcessing_AudioMP3 = True
formats.StringAppend($"--audio-format {aac}", " ")
atCodec = aac
Else
formats.StringAppend($"--audio-format {OutputAudioCodec.StringToLower}", " ")
atCodec = OutputAudioCodec.StringToLower
@@ -716,7 +771,8 @@ Namespace API.YouTube.Objects
If Not cmd.IsEmptyString Then
'URGENT: 2023.3.4 -> 2023.7.6
'cmd = $"yt-dlp -f ""{cmd}"""
cmd = $"yt-dlp -f {cmd}"
'cmd = $"yt-dlp -f {cmd}"
cmd = $"{YTDLP_NAME} -f {cmd}"
If Not MyYouTubeSettings.ReplaceModificationDate Then cmd &= " --no-mtime"
cmd.StringAppend(formats, " ")
cmd.StringAppend(subs, " ")
@@ -738,7 +794,7 @@ Namespace API.YouTube.Objects
_SubtitlesDelegated = New List(Of Subtitles)
SubtitlesSelectedIndexes = New List(Of Integer)
MediaObjects = New List(Of MediaObject)
Files = New List(Of SFile)
_Files = New List(Of SFile)
PostProcessing_OutputSubtitlesFormats = New List(Of String)
PostProcessing_OutputSubtitlesFormats.ListAddList(MyYouTubeSettings.DefaultSubtitlesFormatAddit)
@@ -767,9 +823,19 @@ Namespace API.YouTube.Objects
If RemoveFiles Then
Dim fErr As New ErrorsDescriber(EDP.None)
Dim dMode As SFODelete = SFODelete.DeleteToRecycleBin
Dim paths As New List(Of SFile)
Dim l As New ListAddParams(LAP.NotContainsOnly) With {.Comparer = New FComparer(Of SFile)(Function(x, y) x.PathNoSeparator = y.PathNoSeparator)}
Dim isArr As Boolean = ObjectType <> YouTubeMediaType.Single And ObjectType <> YouTubeMediaType.Undefined
If isArr And AbsolutePath Then paths.ListAddValue(File, l)
File.Delete(SFO.File, dMode, fErr)
If isArr Then paths.ListAddValue(ThumbnailFile, l)
ThumbnailFile.Delete(SFO.File, dMode, fErr)
If Files.Count > 0 Then Files.ForEach(Sub(f) f.Delete(SFO.File, dMode, fErr))
If Files.Count > 0 Then
If isArr Then paths.ListAddList(Files, l)
Files.ForEach(Sub(f) f.Delete(SFO.File, dMode, fErr))
End If
If paths.Count > 0 Then paths.ForEach(Sub(p) If SFile.GetFiles(p,, IO.SearchOption.AllDirectories, EDP.ReturnValue).Count = 0 Then _
p.Delete(SFO.Path, dMode, EDP.SendToLog))
End If
If HasElements Then Elements.ForEach(Sub(e) e.Delete(RemoveFiles))
End Sub
@@ -854,17 +920,22 @@ Namespace API.YouTube.Objects
If HasElements AndAlso Elements(0).ObjectType = YouTubeMediaType.Single AndAlso Elements(0).IsMusic Then
Dim t As TextSaver = Nothing
Try
Dim f As SFile
If MyYouTubeSettings.MusicPlaylistCreate_M3U8 Then
t = New TextSaver
t.AppendLine("#EXTM3U")
Elements.ForEach(Sub(e) t.AppendLine(GetPlaylistRow(e)))
t.SaveAs($"{Elements(0).File.PathWithSeparator}Playlist.m3u8", EDP.SendToLog)
f = $"{Elements(0).File.PathWithSeparator}Playlist.m3u8"
t.SaveAs(f, EDP.SendToLog)
If f.Exists Then AddFile(f)
t.Dispose()
End If
If MyYouTubeSettings.MusicPlaylistCreate_M3U Then
t = New TextSaver
Elements.ForEach(Sub(e) t.AppendLine(e.File))
t.SaveAs($"{Elements(0).File.PathWithSeparator}Playlist.m3u", EDP.SendToLog)
f = $"{Elements(0).File.PathWithSeparator}Playlist.m3u"
t.SaveAs(f, EDP.SendToLog)
If f.Exists Then AddFile(f)
t.Dispose()
End If
Catch ex As Exception
@@ -941,7 +1012,7 @@ Namespace API.YouTube.Objects
ff.Name = "album"
ff.Extension = "url"
CreateUrlFile(url, ff)
If ff.Exists Then Files.Add(ff)
If ff.Exists Then AddFile(ff)
End If
If MyYouTubeSettings.CreateThumbnails_Music Then
Using resp As New Responser
@@ -967,7 +1038,7 @@ Namespace API.YouTube.Objects
url = LinkFormatterSecure(u)
f.Name = "cover"
f.Extension = "jpg"
If resp.DownloadFile(url, f, EDP.ReturnValue) And f.Exists Then CoverDownloaded = True
If resp.DownloadFile(url, f, EDP.ReturnValue) And f.Exists Then CoverDownloaded = True : AddFile(f)
End If
End If
End Using
@@ -976,19 +1047,61 @@ Namespace API.YouTube.Objects
ErrorsDescriber.Execute(EDP.SendToLog, ex, $"DownloadPlaylistCover({PlsId}, {f})")
End Try
End Sub
Private Structure TempFileConversion
Friend File As SFile
Friend Requested As Boolean
Friend ToReplace As Boolean
Friend ReadOnly Property Exists As Boolean
Get
Return File.Exists
End Get
End Property
Friend Sub Delete()
If Not Requested Then File.Delete()
End Sub
Private Sub New(ByVal f As SFile)
File = f
Requested = False
ToReplace = False
End Sub
Friend Sub New(ByVal f As SFile, ByVal Source As YouTubeMediaContainerBase)
Me.New(f)
Requested = Source.PostProcessing_OutputAudioFormats.Count > 0 AndAlso
Source.PostProcessing_OutputAudioFormats.Exists(Function(af) af.StringToLower = f.Extension)
End Sub
Public Shared Widening Operator CType(ByVal f As SFile) As TempFileConversion
Return New TempFileConversion(f)
End Operator
Public Shared Widening Operator CType(ByVal f As TempFileConversion) As SFile
Return f.File
End Operator
Public Overrides Function Equals(ByVal Obj As Object) As Boolean
If Not IsNothing(Obj) Then
If TypeOf Obj Is TempFileConversion Then
Return DirectCast(Obj, TempFileConversion).File = File
ElseIf TypeOf Obj Is SFile Then
Return DirectCast(Obj, SFile) = File
ElseIf TypeOf Obj Is String Then
Return New TempFileConversion(CStr(Obj)).File = File
End If
End If
Return False
End Function
End Structure
Protected Sub DownloadCommand(ByVal UseCookies As Boolean, ByVal Token As CancellationToken)
Dim dCommand$ = String.Empty
Try
ThrowAny(Token)
If MediaState = UMStates.Downloaded Or Not Checked Then Exit Sub
Dim h As DataReceivedEventHandler = Sub(ByVal Sender As Object, ByVal e As DataReceivedEventArgs)
If Not e.Data.IsEmptyString Then
Dim v# = AConvert(Of Double)(RegexReplace(e.Data, DownloadProgressPattern), NumberProvider, -1)
If v >= 0 Then Progress.Value = v : Progress.Perform(0)
End If
End Sub
RaiseEvent FileDownloadStarted(Me, Nothing)
Using batch As New BatchExecutor(True) With {.Encoding = 65001}
Dim h As DataReceivedEventHandler = Sub(ByVal Sender As Object, ByVal e As DataReceivedEventArgs)
If Not e.Data.IsEmptyString Then
Dim v# = AConvert(Of Double)(RegexReplace(e.Data, DownloadProgressPattern), NumberProvider, -1)
If v >= 0 Then Progress.Value = v : Progress.Perform(0)
If Token.IsCancellationRequested Then batch.Kill()
End If
End Sub
With batch
Dim prExists As Boolean = Not Progress Is Nothing
If prExists Then
@@ -1001,7 +1114,7 @@ Namespace API.YouTube.Objects
.Information = $"Download {MediaType}"
End With
End If
.MainProcessName = "yt-dlp"
.MainProcessName = MyYouTubeSettings.YTDLP.Name '"yt-dlp"
.FileExchanger = MyCache.NewInstance(Of BatchFileExchanger)(CachePath, EDP.ReturnValue)
.FileExchanger.DeleteCacheOnDispose = True
.AddCommand("chcp 65001")
@@ -1027,14 +1140,14 @@ Namespace API.YouTube.Objects
Dim fileUrl As SFile = File
fileUrl.Extension = "url"
CreateUrlFile(URL, fileUrl)
If fileUrl.Exists Then Files.Add(fileUrl)
If fileUrl.Exists Then AddFile(fileUrl)
End If
If MyYouTubeSettings.CreateDescriptionFiles And Not Description.IsEmptyString Then
Dim fileDesr As SFile = File
fileDesr.Extension = "txt"
TextSaver.SaveTextToFile(Description, fileDesr,,, EDP.None)
If fileDesr.Exists Then Files.Add(fileDesr)
If fileDesr.Exists Then AddFile(fileDesr)
End If
If PlaylistCount > 0 And Not CoverDownloaded And Not PlaylistID.IsEmptyString Then DownloadPlaylistCover(PlaylistID, File, UseCookies)
@@ -1058,22 +1171,63 @@ Namespace API.YouTube.Objects
Dim format$
Dim fPattern$ = $"{File.PathWithSeparator}{File.Name}." & "{0}"
Dim fPatternFiles$ = $"{File.Name}*." & "{0}"
Dim fAacAudio As New SFile(String.Format(fPattern, aac))
Dim fAc3Audio As New SFile(String.Format(fPattern, ac3))
Dim aacRequested As Boolean = PostProcessing_OutputAudioFormats.Count > 0 AndAlso
PostProcessing_OutputAudioFormats.Exists(Function(af) af.StringToLower = aac)
Dim ac3Requested As Boolean = PostProcessing_OutputAudioFormats.Count > 0 AndAlso
PostProcessing_OutputAudioFormats.Exists(Function(af) af.StringToLower = ac3)
Dim fAacAudio As New TempFileConversion(New SFile(String.Format(fPattern, aac)), Me)
Dim mp3ThumbEmbedded As Boolean = False
Dim tempFilesList As New List(Of TempFileConversion)
Dim ttFile As TempFileConversion
Dim __updateBitrate As Boolean = OutputAudioBitrate > 0 AndAlso (SelectedAudioIndex = -1 OrElse SelectedAudio.Bitrate <> OutputAudioBitrate)
If __updateBitrate Then Bitrate = OutputAudioBitrate
Dim updateBitrate As Action(Of SFile) =
Sub(ByVal sourceFile As SFile)
If __updateBitrate AndAlso sourceFile.Exists Then
Dim destFile As SFile = sourceFile
destFile.Name &= "_new00"
.Execute($"ffmpeg -i ""{sourceFile}"" -crf {MyYouTubeSettings.DefaultAudioBitrate_crf.Value} -b:a {OutputAudioBitrate}k ""{destFile}""")
If destFile.Exists AndAlso sourceFile.Delete Then SFile.Rename(destFile, sourceFile)
End If
End Sub
Dim __getAAC_tried As Boolean = False
Dim AACExists As Func(Of Boolean) = Function() As Boolean
If Not __getAAC_tried Then
__getAAC_tried = True
.Execute($"ffmpeg -i ""{File}"" -vn -acodec {aac} ""{fAacAudio.File}""")
tempFilesList.Add(fAacAudio)
updateBitrate.Invoke(fAacAudio.File)
End If
Return fAacAudio.Exists
End Function
Dim tryToConvert As Action(Of String, SFile) =
Sub(ByVal codec As String, ByVal dFile As SFile)
ThrowAny(Token)
.Execute($"ffmpeg -i ""{File}"" -vn -acodec {codec} ""{dFile}""")
If Not codec = aac AndAlso Not dFile.Exists AndAlso AACExists.Invoke Then
ThrowAny(Token)
.Execute($"ffmpeg -i ""{fAacAudio.File}"" -f {codec} ""{dFile}""")
End If
End Sub
Dim embedThumbTo As Action(Of SFile) =
Sub(ByVal dFile As SFile)
If dFile.Exists And ThumbnailFile.Exists Then
Dim dFileNew As SFile = dFile
dFileNew.Name &= "_NEW"
.Execute($"ffmpeg -i ""{dFile}"" -i ""{ThumbnailFile}"" -map 0:0 -map 1:0 -c copy -id3v2_version 3 -metadata:s:v title=""Cover"" -metadata:s:v comment=""Cover"" ""{dFileNew}""")
If dFileNew.Exists AndAlso dFile.Delete(,, EDP.ReturnValue) Then SFile.Rename(dFileNew, dFile)
End If
End Sub
'Subtitles
ThrowAny(Token)
If PostProcessing_OutputSubtitlesFormats.Count > 0 Then
files = SFile.GetFiles(File, String.Format(fPatternFiles, OutputSubtitlesFormat.StringToLower),, EDP.ReturnValue)
AddFile(files)
If files.ListExists Then
For Each f In files
For Each format In PostProcessing_OutputSubtitlesFormats
format = format.StringToLower
commandFile = $"{f.PathWithSeparator}{f.Name}.{format}"
Me.Files.Add(commandFile)
AddFile(commandFile)
ThrowAny(Token)
.Execute($"ffmpeg -i ""{f}"" ""{commandFile}""")
Next
@@ -1081,46 +1235,81 @@ Namespace API.YouTube.Objects
End If
End If
'Audio
ThrowAny(Token)
If PostProcessing_OutputAudioFormats.Count > 0 Or PostProcessing_AudioAC3 Then
If Not fAacAudio.Exists Then .Execute($"ffmpeg -i ""{File}"" -vn -acodec {aac} ""{fAacAudio}""")
If PostProcessing_AudioAC3 And Not fAc3Audio.Exists Then
ThrowAny(Token)
.Execute($"ffmpeg -i ""{File}"" -vn -acodec {ac3} ""{fAc3Audio}""")
If Not fAc3Audio.Exists And fAacAudio.Exists Then ThrowAny(Token) : .Execute($"ffmpeg -i ""{fAacAudio}"" -f {ac3} ""{fAc3Audio}""")
If PostProcessing_OutputAudioFormats.Count > 0 Or PostProcessing_AudioAC3 Or PostProcessing_AudioMP3 Or __updateBitrate Then
If PostProcessing_AudioAC3 Then
ttFile = New TempFileConversion(New SFile(String.Format(fPattern, ac3)), Me) With {.ToReplace = True}
tempFilesList.Add(ttFile)
If Not ttFile.Exists Then tryToConvert.Invoke(ac3, ttFile.File)
updateBitrate.Invoke(ttFile.File)
End If
If PostProcessing_AudioMP3 Then
ttFile = New TempFileConversion(New SFile(String.Format(fPattern, mp3)), Me) With {.ToReplace = True}
tempFilesList.Add(ttFile)
If Not ttFile.Requested Then ttFile.Requested = SelectedVideoIndex = -1 And OutputAudioCodec.StringToLower = mp3
If Not ttFile.Exists Then tryToConvert.Invoke(mp3, ttFile.File)
updateBitrate.Invoke(ttFile.File)
embedThumbTo.Invoke(ttFile.File)
mp3ThumbEmbedded = True
End If
If __updateBitrate Then
format = OutputAudioCodec.StringToLower
If Not format.IsEmptyString Then
f = String.Format(fPattern, format)
ttFile = New TempFileConversion(f, Me) With {.ToReplace = True}
If Not ttFile.Requested Then ttFile.Requested = SelectedVideoIndex = -1
If Not f.Exists Then
tempFilesList.ListAddValue(ttFile, LAP.NotContainsOnly)
tryToConvert.Invoke(format, f)
updateBitrate.Invoke(f)
ElseIf Not tempFilesList.Contains(ttFile) Then
tempFilesList.Add(ttFile)
updateBitrate.Invoke(f)
End If
End If
End If
If PostProcessing_OutputAudioFormats.Count > 0 Then
For Each format In PostProcessing_OutputAudioFormats
format = format.StringToLower
f = String.Format(fPattern, format)
Me.Files.Add(f)
If Not format = ac3 Or Not f.Exists Then
ThrowAny(Token)
.Execute($"ffmpeg -i ""{fAacAudio}"" -f {format} ""{f}""")
AddFile(f)
If Not f.Exists Then
tryToConvert.Invoke(format, f)
updateBitrate(f)
If format = mp3 And Not mp3ThumbEmbedded And MyYouTubeSettings.DefaultAudioEmbedThumbnail_ExtractedFiles Then _
embedThumbTo.Invoke(f) : mp3ThumbEmbedded = True
If Not M3U8_PlaylistFiles.ListExists AndAlso f.Exists Then M3U8_Append(f)
End If
Next
End If
End If
'Update video
ThrowAny(Token)
If PostProcessing_AudioAC3 Then
If SelectedVideoIndex >= 0 AndAlso tempFilesList.Count > 0 AndAlso tempFilesList.Exists(Function(tf) tf.ToReplace) Then
f = File
If SelectedVideoIndex >= 0 Then
f.Name &= "tmp00"
Else
f.Extension = ac3
f.Name &= "tmp00"
Dim tfr As SFile = tempFilesList.FirstOrDefault(Function(tf) tf.ToReplace).File
If tfr.Exists And Not f.Exists Then
ThrowAny(Token)
.Execute($"ffmpeg -i ""{File}"" -i ""{tfr}"" -c:v copy -c copy -map 0:v:0 -map 1:a:0 ""{f}""")
End If
If Not f.Exists Then ThrowAny(Token) : .Execute($"ffmpeg -i ""{File}"" -i ""{fAc3Audio}"" -c:v copy -c copy -map 0:v:0 -map 1:a:0 ""{f}""")
If f.Exists Then
File.Delete()
If SelectedVideoIndex >= 0 Then SFile.Rename(f, File,, EDP.LogMessageValue)
End If
If fAacAudio.Exists And Not aacRequested Then fAacAudio.Delete()
If fAc3Audio.Exists And Not ac3Requested And SelectedVideoIndex >= 0 Then fAc3Audio.Delete()
End If
If SelectedVideoIndex >= 0 AndAlso OutputVideoFPS > 0 AndAlso SelectedVideo.Bitrate > OutputVideoFPS Then
'Delete unrequsted files
If tempFilesList.Count > 0 Then tempFilesList.ForEach(Sub(tfr) If Not tfr.Requested Then tfr.File.Delete(,, EDP.None)) : tempFilesList.Clear()
'Update video FPS
If SelectedVideoIndex >= 0 AndAlso OutputVideoFPS > 0 AndAlso SelectedVideo.Bitrate <> OutputVideoFPS Then
f = File
f.Name &= "tmp00"
.Execute($"ffmpeg -i ""{File}"" -filter:v fps={OutputVideoFPS.ToString.Replace(",", ".")} -c:a copy ""{f}""")
@@ -1132,6 +1321,12 @@ Namespace API.YouTube.Objects
End If
End If
End With
Dim newSize# = 0
If File.Exists Then newSize += File.Size
If Files.Count > 0 Then newSize += (From eFile As SFile In Files Where eFile.Exists Select eFile.Size).Sum
If ThumbnailFile.Exists Then newSize += ThumbnailFile.Size
If newSize > 0 Then newSize /= 1024 : Size = newSize : _SizeRecalculated = True
End Using
_MediaState = UMStates.Downloaded
Catch oex As OperationCanceledException When Token.IsCancellationRequested
@@ -1304,6 +1499,7 @@ Namespace API.YouTube.Objects
End Sub
#End Region
#Region "Parse"
Friend Const DRC As String = "drc"
Public Overridable Function Parse(ByVal Container As EContainer, ByVal Path As SFile, ByVal IsMusic As Boolean,
Optional ByVal Token As CancellationToken = Nothing, Optional ByVal Progress As IMyProgress = Nothing) As Boolean Implements IYouTubeMediaContainer.Parse
Try
@@ -1366,6 +1562,7 @@ Namespace API.YouTube.Objects
_File.Name = $"{ID}.{ext}"
End If
If Not MyYouTubeSettings.OutputPath.IsEmptyString Then _File.Path = MyYouTubeSettings.OutputPath.Value.Path
_File = CleanFileName(_File)
File = _File
If .Contains("duration") Then
@@ -1455,7 +1652,8 @@ Namespace API.YouTube.Objects
obj = New MediaObject With {
.ID = ee.Value("format_id"),
.URL = ee.Value("url"),
.Extension = ee.Value("ext")
.Extension = ee.Value("ext"),
.ID_DRC = Not .ID.IsEmptyString AndAlso .ID.StringToLower.Contains(DRC)
}
obj.Width = AConvert(Of Integer)(ee.Value("width"), NumberProvider, -1)
obj.Height = AConvert(Of Integer)(ee.Value("height"), NumberProvider, -1)
@@ -1510,6 +1708,14 @@ Namespace API.YouTube.Objects
If MediaObjects.Count > 0 AndAlso MediaObjects.LongCount(CountAVC) > 0 Then MediaObjects.RemoveAll(RemoveAVC)
Next
End If
If t = UMTypes.Audio And MediaObjects.Count > 0 Then
Dim __audioComparerCount As Func(Of MediaObject, MediaObject, Boolean) =
Function(mo, mo2) (mo2.Type = t And mo2.Extension = mo.Extension And mo2.Bitrate = mo.Bitrate) AndAlso
mo2.Size.RoundDown = mo.Size.RoundDown AndAlso ACheck(Of Integer)(mo2.ID)
Dim RemoveDRC As Predicate(Of MediaObject) = Function(mo) mo.Type = t AndAlso Not ACheck(Of Integer)(mo.ID) AndAlso
MediaObjects.LongCount(Function(mo2) __audioComparerCount.Invoke(mo, mo2)) > 0
MediaObjects.RemoveAll(RemoveDRC)
End If
End Sub
Dim protocolCleaner As Action =
Sub()
@@ -1691,7 +1897,7 @@ Namespace API.YouTube.Objects
_SubtitlesDelegated.Clear()
SubtitlesSelectedIndexes.Clear()
MediaObjects.Clear()
Files.Clear()
_Files.Clear()
PostProcessing_OutputAudioFormats.Clear()
PostProcessing_OutputSubtitlesFormats.Clear()
End If

2
SCrawler/API/Base/Declarations.vb
View File

@@ -49,7 +49,7 @@ Namespace API.Base
End Sub
Public Overrides Function Convert(ByVal Value As Object, ByVal DestinationType As Type, ByVal Provider As IFormatProvider,
Optional ByVal NothingArg As Object = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Object
Dim v% = AConvert(Of Integer)(Value, -1)
Dim v% = AConvert(Of Integer)(Value, -1, EDP.ReturnValue)
If v > 0 Then
Return Value
ElseIf Not ACheck(Of Integer)(Value) Then

2
SCrawler/API/Base/DeclaredNames.vb
View File

@@ -11,8 +11,6 @@ Namespace API.Base
Friend Const Header_Authorization As String = "authorization"
Friend Const Header_CSRFToken As String = "x-csrf-token"
Friend Const Header_FB_FRIENDLY_NAME As String = "x-fb-friendly-name"
Friend Const ConcurrentDownloadsCaption As String = "Concurrent downloads"
Friend Const ConcurrentDownloadsToolTip As String = "The number of concurrent downloads."
Friend Const SavedPostsUserNameCaption As String = "Saved posts user"

24
SCrawler/API/Base/M3U8Base.vb
View File

@@ -42,6 +42,13 @@ Namespace API.Base
Friend NotInheritable Class M3U8Base
Friend Const TempCacheFolderName As String = "tmpCache"
Friend Const TempFilePrefix As String = "ConPart_"
Friend Const TempFileDefaultExtension As String = "ts"
''' <summary><c>SFileNumbers.NumberProviderDefault</c></summary>
Friend Shared ReadOnly Property NumberProviderDefault As ANumbers
Get
Return SFileNumbers.NumberProviderDefault
End Get
End Property
Private Sub New()
End Sub
Friend Shared Function CreateUrl(ByVal Appender As String, ByVal File As String) As String
@@ -63,8 +70,7 @@ Namespace API.Base
Friend Overloads Shared Function Download(ByVal URLs As List(Of M3U8URL), ByVal DestinationFile As SFile, Optional ByVal Responser As Responser = Nothing,
Optional ByVal Token As CancellationToken = Nothing, Optional ByVal Progress As MyProgress = Nothing,
Optional ByVal UsePreProgress As Boolean = True, Optional ByVal ExistingCache As CacheKeeper = Nothing,
Optional ByVal OnlyDownload As Boolean = False) As SFile
Const defaultExtension$ = "ts"
Optional ByVal OnlyDownload As Boolean = False, Optional ByVal SkipBroken As Boolean = False) As SFile
Dim Cache As CacheKeeper = Nothing
Using tmpPr As New PreProgress(Progress)
Try
@@ -89,13 +95,13 @@ Namespace API.Base
End If
End If
Dim p As SFileNumbers = SFileNumbers.Default(ConcatFile.Name)
Dim pNum As ANumbers = SFileNumbers.NumberProviderDefault
Dim pNum As ANumbers = NumberProviderDefault
p.NumberProvider = pNum
DirectCast(p.NumberProvider, ANumbers).GroupSize = {URLs.Count.ToString.Length, 3}.Max
ConcatFile = SFile.IndexReindex(ConcatFile,,, p, EDP.ReturnValue)
Dim i%
Dim dFile As SFile = cache2.RootDirectory
dFile.Extension = defaultExtension
dFile.Extension = TempFileDefaultExtension
Using w As New DownloadObjects.WebClient2(Responser)
For i = 0 To URLs.Count - 1
If progressExists Then
@@ -107,9 +113,13 @@ Namespace API.Base
End If
Token.ThrowIfCancellationRequested()
dFile.Name = $"{TempFilePrefix}{i.NumToString(pNum)}"
dFile.Extension = URLs(i).Extension.IfNullOrEmpty(defaultExtension)
w.DownloadFile(URLs(i).URL, dFile)
cache2.AddFile(dFile, True)
dFile.Extension = URLs(i).Extension.IfNullOrEmpty(TempFileDefaultExtension)
Try
w.DownloadFile(URLs(i).URL, dFile)
cache2.AddFile(dFile, True)
Catch ex As Exception
If Not SkipBroken Then Throw ex
End Try
Next
End Using
If Not OnlyDownload Then _

5
SCrawler/API/Base/SiteSettingsBase.vb
View File

@@ -17,6 +17,7 @@ Imports Download = SCrawler.Plugin.ISiteSettings.Download
Namespace API.Base
Friend MustInherit Class SiteSettingsBase : Implements ISiteSettings, IResponserContainer
#Region "Declarations"
<PXML> Protected ReadOnly Property SettingsVersion As PropertyValue
Friend ReadOnly Property Site As String Implements ISiteSettings.Site
Protected _Icon As Icon = Nothing
Friend Overridable ReadOnly Property Icon As Icon Implements ISiteSettings.Icon
@@ -61,6 +62,7 @@ Namespace API.Base
End Sub
#End Region
#Region "Responser and cookies support"
Friend Const ResponserFilePrefix As String = "Responser_"
Private _CookiesNetscapeFile As SFile = Nothing
Friend ReadOnly Property CookiesNetscapeFile As SFile
Get
@@ -91,7 +93,7 @@ Namespace API.Base
End Property
Protected Sub UpdateResponserFile()
Dim acc$ = If(AccountName.IsEmptyString OrElse AccountName = Hosts.SettingsHost.NameAccountNameDefault, String.Empty, $"_{AccountName}")
Responser.File = $"{SettingsFolderName}\Responser_{Site}{acc}.xml"
Responser.File = $"{SettingsFolderName}\{ResponserFilePrefix}{Site}{acc}.xml"
_CookiesNetscapeFile = Responser.File
_CookiesNetscapeFile.Name &= "_Cookies_Netscape"
_CookiesNetscapeFile.Extension = "txt"
@@ -106,6 +108,7 @@ Namespace API.Base
_Icon = __Icon
_Image = __Image
Responser = New Responser With {.DeclaredError = EDP.ThrowException}
SettingsVersion = New PropertyValue(0)
UpdateResponserFile()
End Sub
Friend Sub New(ByVal SiteName As String, ByVal CookiesDomain As String, ByVal AccName As String, ByVal Temp As Boolean,

7
SCrawler/API/Base/UserDataBase.vb
View File

@@ -1165,7 +1165,7 @@ BlockNullPicture:
If Not Responser Is Nothing Then Responser.Dispose()
Responser = New Responser
If Not HOST.Responser Is Nothing Then Responser.Copy(HOST.Responser)
If Not Responser Is Nothing And _ResponserAutoUpdateCookies Then
If Not Responser Is Nothing And (_ResponserAutoUpdateCookies Or _ResponserAddResponseReceivedHandler) Then
If _ResponserAutoUpdateCookies Then
Responser.CookiesUpdateMode = CookieUpdateModes.ReplaceByNameAll
Responser.CookiesExtractMode = Responser.CookiesExtractModes.Any
@@ -1339,6 +1339,7 @@ BlockNullPicture:
ResetHost()
URL = Data.URL
AccountName = Data.AccountName
TokenQueue = Token
If HOST Is Nothing Then Throw New ExitException($"Host '{AccountName}' not found")
Data.DownloadState = UserMediaStates.Tried
Progress = Data.Progress
@@ -1399,6 +1400,9 @@ BlockNullPicture:
f = Web.FFMPEG.TakeSnapshot(f, ff, Settings.FfmpegFile, TimeSpan.FromSeconds(1),,, EDP.SendToLog + EDP.ReturnValue)
If f.Exists Then DirectCast(Data, IDownloadableMedia).ThumbnailFile = f
End If
Dim filesSize# = (From mm As UserMedia In _ContentNew Where mm.State = UStates.Downloaded AndAlso mm.File.Exists Select mm.File.Size).Sum
If filesSize > 0 Then filesSize /= 1024
Data.Size = filesSize
Else
Data.DownloadState = UserMediaStates.Missing
End If
@@ -1771,6 +1775,7 @@ BlockNullPicture:
Protected Overridable Function ValidateDownloadFile(ByVal URL As String, ByVal Media As UserMedia, ByRef Interrupt As Boolean) As Boolean
Return True
End Function
''' <returns><c>MyFile.CutPath(IIf(IsSingleObjectDownload, 0, 1)).PathNoSeparator</c></returns>
Protected Overridable Function DownloadContentDefault_GetRootDir() As String
Return MyFile.CutPath(IIf(IsSingleObjectDownload, 0, 1)).PathNoSeparator
End Function

2
SCrawler/API/Base/YTDLP.vb
View File

@@ -11,7 +11,7 @@ Namespace API.Base.YTDLP
Friend Sub New(ByVal _Token As Threading.CancellationToken)
MyBase.New(_Token)
Commands.Clear()
MainProcessName = "yt-dlp"
MainProcessName = Settings.YtdlpFile.File.Name '"yt-dlp"
ChangeDirectory(Settings.YtdlpFile.File)
End Sub
End Class

4
SCrawler/API/Facebook/SiteSettings.vb
View File

@@ -46,7 +46,7 @@ Namespace API.Facebook
With Responser.Headers
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Authority, "www.facebook.com"))
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.Origin, "https://www.facebook.com"))
.Remove(DeclaredNames.Header_FB_FRIENDLY_NAME)
.Remove(Instagram.UserData.GQL_HEADER_FB_FRINDLY_NAME)
End With
Header_Accept = New PropertyValue(String.Empty, GetType(String))
ParsePhotoBlock = New PropertyValue(True)
@@ -74,7 +74,7 @@ Namespace API.Facebook
#End Region
#Region "BaseAuthExists, GetUserUrl, GetUserPostUrl, IsMyUser, IsMyImageVideo"
Friend Overrides Function BaseAuthExists() As Boolean
Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value)
Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value) And CBool(DownloadData_Impl.Value)
End Function
Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String
Return DirectCast(User, UserData).GetProfileUrl

81
SCrawler/API/Facebook/UserData.vb
View File

@@ -124,26 +124,34 @@ Namespace API.Facebook
.SendToLogOnlyMessage = True, .ReplaceMainMessage = True})
End Sub
End Class
Private Token_dtsg As String = String.Empty
Private Token_lsd As String = String.Empty
Private Token_Photosby As String = String.Empty
Private Limit As Integer = -1
Private Sub WaitTimer()
If CInt(MySettings.RequestsWaitTimer_Any.Value) > 0 Then Thread.Sleep(CInt(MySettings.RequestsWaitTimer_Any.Value))
End Sub
Private Sub DisableDownload()
MySettings.DownloadData_Impl.Value = False
MyMainLOG = $"{Site} downloading is disabled until you update your credentials"
End Sub
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
Try
GetUserTokens(Token)
LoadSavePostsKV(True)
Limit = If(DownloadTopCount, -1)
If IsSavedPosts Then
DownloadData_SavedPosts(String.Empty, Token)
Else
If DownloadImages And ParsePhotoBlock Then DownloadData_Photo(String.Empty, Token)
If DownloadVideos And ParseVideoBlock Then DownloadData_Video(String.Empty, Token)
If (DownloadImages Or DownloadVideos) And ParseStoriesBlock Then DownloadData_Stories(Token)
End If
LoadSavePostsKV(False)
Finally
MySettings.UpdateResponserData(Responser)
End Try
If CBool(MySettings.DownloadData_Impl.Value) Then
Try
ResetBaseTokens()
GetUserTokens(Token)
LoadSavePostsKV(True)
Limit = If(DownloadTopCount, -1)
If IsSavedPosts Then
DownloadData_SavedPosts(String.Empty, Token)
Else
If DownloadImages And ParsePhotoBlock Then DownloadData_Photo(String.Empty, Token)
If DownloadVideos And ParseVideoBlock Then DownloadData_Video(String.Empty, Token)
If (DownloadImages Or DownloadVideos) And ParseStoriesBlock Then DownloadData_Stories(Token)
End If
LoadSavePostsKV(False)
Finally
MySettings.UpdateResponserData(Responser)
End Try
End If
End Sub
Private Const Header_fb_fr_name_Photo As String = "ProfileCometAppCollectionPhotosRendererPaginationQuery"
Private Const Header_fb_fr_name_Video As String = "PagesCometChannelTabAllVideosCardImplPaginationQuery"
@@ -167,13 +175,13 @@ Namespace API.Facebook
ValidateBaseTokens()
If Token_Photosby.IsEmptyString Then Throw New TokensException("Unable to obtain token 'Token_Photosby'", False)
URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Photo, Header_fb_fr_name_Photo,
SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg),
URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Photo, Header_fb_fr_name_Photo, Token_dtsg_Var,
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, Cursor, Token_Photosby) & "}"))
ResponserApplyDefs(Header_fb_fr_name_Photo)
ThrowAny(Token)
WaitTimer()
Dim r$ = Responser.GetResponse(URL)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r)
@@ -233,13 +241,13 @@ Namespace API.Facebook
If VideoPageID.IsEmptyString Then Throw New TokensException("Unable to obtain 'VideoPageID'", False)
ValidateBaseTokens()
URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Video, Header_fb_fr_name_Video,
SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg),
URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Video, Header_fb_fr_name_Video, Token_dtsg_Var,
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, If(Cursor.IsEmptyString, "null", $"""{Cursor}"""), VideoPageID) & "}"))
ResponserApplyDefs(Header_fb_fr_name_Video)
ThrowAny(Token)
WaitTimer()
Dim r$ = Responser.GetResponse(URL)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r)
@@ -288,13 +296,13 @@ Namespace API.Facebook
ValidateBaseTokens()
If StoryBucket.IsEmptyString Then Throw New TokensException("Unable to obtain 'StoryBucket'", False)
URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Stories, Header_fb_fr_name_Stories,
SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg),
URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_Stories, Header_fb_fr_name_Stories, Token_dtsg_Var,
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, StoryBucket) & "}"))
ResponserApplyDefs(Header_fb_fr_name_Stories)
ThrowAny(Token)
WaitTimer()
Dim r$ = Responser.GetResponse(URL)
If Not r.IsEmptyString Then r = RegexReplace(r, RParams.DM("[^\r\n]+", 0, EDP.ReturnValue))
If Not r.IsEmptyString Then
@@ -357,13 +365,13 @@ Namespace API.Facebook
Dim pid As PostKV
ValidateBaseTokens()
URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_SavedPosts, Header_fb_fr_name_SavedPosts,
SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg),
URL = String.Format(Graphql_UrlPattern, Token_lsd, DocID_SavedPosts, Header_fb_fr_name_SavedPosts, Token_dtsg_Var,
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(VarPattern, If(Cursor.IsEmptyString, "null", $"""{Cursor}""")) & "}"))
ResponserApplyDefs(Header_fb_fr_name_SavedPosts)
ThrowAny(Token)
WaitTimer()
Dim r$ = Responser.GetResponse(URL)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r)
@@ -421,6 +429,7 @@ Namespace API.Facebook
If Round > 0 Then ThrowAny(Token)
Dim script$, newUrl$
Dim jNode As EContainer, jNode2 As EContainer
WaitTimer()
Dim r$ = resp.GetResponse(PostUrl)
If Not r.IsEmptyString Then
@@ -488,16 +497,20 @@ Namespace API.Facebook
#End Region
#Region "ValidateBaseTokens, GetVideoPageID, GetUserTokens"
''' <exception cref="ArgumentNullException"></exception>
Private Sub ValidateBaseTokens()
Protected Overrides Function ValidateBaseTokens() As Boolean
Dim tokens$ = String.Empty
If Token_dtsg.IsEmptyString Then tokens.StringAppend("Token_dtsg")
If Token_lsd.IsEmptyString Then tokens.StringAppend("Token_lsd")
If Not tokens.IsEmptyString Then Throw New TokensException($"Unable to obtain token(s) ({tokens}){vbCr}Your credentials may have expired.", True)
End Sub
If Not ValidateBaseTokens(tokens) Then
DisableDownload()
Throw New TokensException($"Unable to obtain token(s) ({tokens}). Your credentials may have expired.", True)
Else
Return True
End If
End Function
Private Sub GetVideoPageID(ByVal Token As CancellationToken)
Dim URL$ = $"{GetProfileUrl()}\videos"
Dim resp As Responser = HtmlResponserCreate()
Try
WaitTimer()
Dim r$ = resp.GetResponse(URL)
If Not r.IsEmptyString Then VideoPageID = RegexReplace(r, Regex_VideoPageID)
Catch ex As Exception
@@ -510,9 +523,9 @@ Namespace API.Facebook
Dim URL$ = If(IsSavedPosts, "https://www.facebook.com/saved", GetProfileUrl())
Dim resp As Responser = HtmlResponserCreate()
Try
Token_dtsg = String.Empty
Token_lsd = String.Empty
ResetBaseTokens()
Token_Photosby = String.Empty
WaitTimer()
Dim r$ = resp.GetResponse(URL)
If Not r.IsEmptyString Then
If Responser.CookiesExists Then Responser.Cookies.Update(resp.Cookies)
@@ -535,8 +548,7 @@ Namespace API.Facebook
#Region "Responser options"
Private Sub ResponserApplyDefs(ByVal __fb_friendly_name As String)
With Responser
.Headers.Add(ThreadsNet.UserData.Header_FB_LSD, Token_lsd)
.Headers.Add(DeclaredNames.Header_FB_FRIENDLY_NAME, __fb_friendly_name)
UpdateHeadersGQL(__fb_friendly_name)
.Method = "POST"
.Accept = "*/*"
.Referer = GetProfileUrl()
@@ -655,6 +667,7 @@ Namespace API.Facebook
Else
URL = String.Format(VideoHtmlUrlPattern, m.Post.ID)
End If
WaitTimer()
r = resp.GetResponse(URL)
If Not r.IsEmptyString Then
re.Pattern = String.Format(pattern, nameHD)

8
SCrawler/API/Instagram/Declarations.vb
View File

@@ -18,7 +18,7 @@ Namespace API.Instagram
Friend ReadOnly ObtainMedia_SizeFuncPic_RegexP As RParams = RParams.DMS("_p(\d+)x(\d+)", 1, EDP.ReturnValue)
Friend ReadOnly ObtainMedia_SizeFuncPic_RegexS As RParams = RParams.DMS("_s(\d+)x(\d+)", 1, EDP.ReturnValue)
Friend Const PageTokenRegexPatternDefault As String = "\[\],{""token"":""(.*?)""},\d+\]"
Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser)
Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser, ByVal UpdateWwwClaim As Boolean)
Const r_wwwClaimName$ = "x-ig-set-www-claim"
Const r_tokenName$ = SiteSettings.Header_CSRF_TOKEN_COOKIE
If Not Source Is Nothing Then
@@ -35,17 +35,17 @@ Namespace API.Instagram
Dim token$ = String.Empty
With Source
If isInternal Then
If .HeadersExists Then wwwClaim = .Headers.Value(wwwClaimName)
If UpdateWwwClaim And .HeadersExists Then wwwClaim = .Headers.Value(wwwClaimName)
If .CookiesExists Then token = If(.Cookies.FirstOrDefault(Function(c) c.Name = tokenName)?.Value, String.Empty)
Else
If .HeadersExists Then
wwwClaim = .Headers.Value(wwwClaimName)
If UpdateWwwClaim Then wwwClaim = .Headers.Value(wwwClaimName)
token = .Headers.Value(tokenName)
End If
End If
End With
If Not wwwClaim.IsEmptyString Then Destination.Headers.Add(SiteSettings.Header_IG_WWW_CLAIM, wwwClaim)
If UpdateWwwClaim And Not wwwClaim.IsEmptyString Then Destination.Headers.Add(SiteSettings.Header_IG_WWW_CLAIM, wwwClaim)
If Not token.IsEmptyString Then Destination.Headers.Add(SiteSettings.Header_CSRF_TOKEN, token)
If Not isInternal Then
Destination.Cookies.Update(Source.Cookies, CookieKeeper.UpdateModes.ReplaceByNameAll, False, EDP.SendToLog)

206
SCrawler/API/Instagram/SiteSettings.vb
View File

@@ -19,7 +19,7 @@ Namespace API.Instagram
Friend Class SiteSettings : Inherits SiteSettingsBase
#Region "Declarations"
#Region "Providers"
Private Class TimersChecker : Inherits FieldsCheckerProviderBase
Friend Class TimersChecker : Inherits FieldsCheckerProviderBase
Private ReadOnly LVProvider As New ANumbers With {.FormatOptions = ANumbers.Options.GroupIntegral}
Private ReadOnly _LowestValue As Integer
Friend Sub New(ByVal LowestValue As Integer)
@@ -32,7 +32,7 @@ Namespace API.Instagram
If Not ACheck(Of Integer)(Value) Then
TypeError = True
ElseIf CInt(Value) < _LowestValue Then
ErrorMessage = $"The value of [{Name}] field must be greater than or equal to {_LowestValue.NumToString(LVProvider)}"
ErrorMessage = $"The value of '{Name}' field must be greater than or equal to {_LowestValue.NumToString(LVProvider)}"
HasError = True
Else
Return Value
@@ -47,7 +47,7 @@ Namespace API.Instagram
If v > 0 Or v = -1 Then
Return Value
Else
ErrorMessage = $"The value of [{Name}] field must be greater than 0 or equal to -1"
ErrorMessage = $"The value of '{Name}' field must be greater than 0 or equal to -1"
HasError = True
Return Nothing
End If
@@ -66,24 +66,30 @@ Namespace API.Instagram
<PropertyOption(ControlText:="x-csrftoken", ControlToolTip:="Can be automatically extracted from cookies", IsAuth:=True, AllowNull:=True), ControlNumber(2), PClonable(Clone:=False)>
Friend ReadOnly Property HH_CSRF_TOKEN As PropertyValue
<PropertyOption(ControlText:="x-ig-app-id", IsAuth:=True, AllowNull:=False), ControlNumber(3), PClonable(Clone:=False)>
Friend Property HH_IG_APP_ID As PropertyValue
Friend ReadOnly Property HH_IG_APP_ID As PropertyValue
<PropertyOption(ControlText:="x-asbd-id", IsAuth:=True, AllowNull:=True), ControlNumber(4), PClonable(Clone:=False)>
Friend Property HH_ASBD_ID As PropertyValue
Friend ReadOnly Property HH_ASBD_ID As PropertyValue
'PropertyOption(ControlText:="x-ig-www-claim", IsAuth:=True, AllowNull:=True)
<ControlNumber(5), PClonable(Clone:=False)>
Friend Property HH_IG_WWW_CLAIM As PropertyValue
Friend ReadOnly Property HH_IG_WWW_CLAIM As PropertyValue
Private ReadOnly Property HH_IG_WWW_CLAIM_IS_ZERO As Boolean
Get
Dim v$ = AConvert(Of String)(HH_IG_WWW_CLAIM.Value, String.Empty)
Return Not v.IsEmptyString AndAlso v = "0"
End Get
End Property
<PropertyOption(ControlText:="sec-ch-ua", IsAuth:=True, AllowNull:=True,
InheritanceName:=SettingsCLS.HEADER_DEF_sec_ch_ua), ControlNumber(6), PClonable, PXML(OnlyForChecked:=True)>
Private Property HH_BROWSER As PropertyValue
Private ReadOnly Property HH_BROWSER As PropertyValue
<PropertyOption(ControlText:="sec-ch-ua-full", ControlToolTip:="sec-ch-ua-full-version-list", IsAuth:=True, AllowNull:=True,
InheritanceName:=SettingsCLS.HEADER_DEF_sec_ch_ua_full_version_list), ControlNumber(7), PClonable, PXML(OnlyForChecked:=True)>
Private Property HH_BROWSER_EXT As PropertyValue
Private ReadOnly Property HH_BROWSER_EXT As PropertyValue
<PropertyOption(ControlText:="sec-ch-ua-platform-ver", ControlToolTip:="sec-ch-ua-platform-version", IsAuth:=True, AllowNull:=True, LeftOffset:=135,
InheritanceName:=SettingsCLS.HEADER_DEF_sec_ch_ua_platform_version), ControlNumber(8), PClonable, PXML(OnlyForChecked:=True)>
Private Property HH_PLATFORM As PropertyValue
Private ReadOnly Property HH_PLATFORM As PropertyValue
<PropertyOption(ControlText:="UserAgent", IsAuth:=True, AllowNull:=True,
InheritanceName:=SettingsCLS.HEADER_DEF_UserAgent), ControlNumber(9), PClonable, PXML(OnlyForChecked:=True)>
Private Property HH_USER_AGENT As PropertyValue
Private ReadOnly Property HH_USER_AGENT As PropertyValue
Friend Overrides Function BaseAuthExists() As Boolean
Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value) And ACheck(HH_CSRF_TOKEN.Value)
End Function
@@ -110,17 +116,55 @@ Namespace API.Instagram
End If
End If
End Sub
#Region "HH_IG_WWW_CLAIM"
<PropertyOption(ControlText:="ig-www-claim update interval", IsAuth:=True, LeftOffset:=150), PXML, ControlNumber(10), PClonable, HiddenControl>
Private ReadOnly Property HH_IG_WWW_CLAIM_UPDATE_INTERVAL As PropertyValue
<PropertyOption(ControlText:="ig-www-claim: always 0", ControlToolTip:="Keep token value always = 0", IsAuth:=True),
PXML, ControlNumber(11), PClonable, HiddenControl>
Friend ReadOnly Property HH_IG_WWW_CLAIM_ALWAYS_ZERO As PropertyValue
<PropertyOption(ControlText:="ig-www-claim: reset each session", ControlToolTip:="Set 'x-ig-www-claim' to '0' before each session", IsAuth:=True),
PXML, ControlNumber(12), PClonable, HiddenControl>
Friend ReadOnly Property HH_IG_WWW_CLAIM_RESET_EACH_SESSION As PropertyValue
<PropertyOption(ControlText:="ig-www-claim: reset each target", ControlToolTip:="Set 'x-ig-www-claim' to '0' before each target", IsAuth:=True),
PXML, ControlNumber(13), PClonable, HiddenControl>
Friend ReadOnly Property HH_IG_WWW_CLAIM_RESET_EACH_TARGET As PropertyValue
<PropertyOption(ControlText:="ig-www-claim: use in requests", IsAuth:=True), PXML, ControlNumber(14), PClonable, HiddenControl>
Friend ReadOnly Property HH_IG_WWW_CLAIM_USE As PropertyValue
<PropertyOption(ControlText:="ig-www-claim: use default algorithm to update", IsAuth:=True), PXML, ControlNumber(15), PClonable, HiddenControl>
Friend ReadOnly Property HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO As PropertyValue
<Provider(NameOf(HH_IG_WWW_CLAIM_UPDATE_INTERVAL), FieldsChecker:=True)>
Private ReadOnly Property TokenUpdateIntervalProvider As IFormatProvider
#End Region
<PropertyOption(ControlText:="Use GraphQL to download", IsAuth:=True), PXML, ControlNumber(16), PClonable>
Friend ReadOnly Property USE_GQL As PropertyValue
#End Region
#Region "Download properties"
<PropertyOption(ControlText:="Request timer", AllowNull:=False), PXML("RequestsWaitTimer"), ControlNumber(20), PClonable>
Friend Const TimersUrgentTip As String = vbCr & "It is highly recommended not to change the default value."
<PropertyOption(ControlText:="Request timer (any)",
ControlToolTip:="The timer (in milliseconds) that SCrawler should wait before executing the next request." &
vbCr & "The default value is 1'000." & vbCr & "The minimum value is 0." & TimersUrgentTip, AllowNull:=False),
PXML, ControlNumber(19), PClonable>
Friend ReadOnly Property RequestsWaitTimer_Any As PropertyValue
<Provider(NameOf(RequestsWaitTimer_Any), FieldsChecker:=True)>
Private ReadOnly Property RequestsWaitTimer_AnyProvider As IFormatProvider
<PropertyOption(ControlText:="Request timer",
ControlToolTip:="The time value (in milliseconds) that the program will wait before processing the next 'Request time counter' request." &
vbCr & "The default value is 1'000." & vbCr & "The minimum value is 100." & TimersUrgentTip,
AllowNull:=False), PXML, ControlNumber(20), PClonable>
Friend ReadOnly Property RequestsWaitTimer As PropertyValue
<Provider(NameOf(RequestsWaitTimer), FieldsChecker:=True)>
Private ReadOnly Property RequestsWaitTimerProvider As IFormatProvider
<PropertyOption(ControlText:="Request timer counter", AllowNull:=False, LeftOffset:=120), PXML("RequestsWaitTimerTaskCount"), ControlNumber(21), PClonable>
<PropertyOption(ControlText:="Request timer counter",
ControlToolTip:="How many requests will be sent to Instagram before the program waits 'Request timer'." &
vbCr & "The default value is 1." & vbCr & "The minimum value is 1." & TimersUrgentTip,
AllowNull:=False, LeftOffset:=120), PXML, ControlNumber(21), PClonable>
Friend ReadOnly Property RequestsWaitTimerTaskCount As PropertyValue
<Provider(NameOf(RequestsWaitTimerTaskCount), FieldsChecker:=True)>
Private ReadOnly Property RequestsWaitTimerTaskCountProvider As IFormatProvider
<PropertyOption(ControlText:="Posts limit timer", AllowNull:=False), PXML("SleepTimerOnPostsLimit"), ControlNumber(22), PClonable>
<PropertyOption(ControlText:="Posts limit timer",
ControlToolTip:="The time value (in milliseconds) the program will wait before processing the next request after 195 requests." &
vbCr & "The default value is 60'000." & vbCr & "The minimum value is 10'000." & TimersUrgentTip,
AllowNull:=False), PXML, ControlNumber(22), PClonable>
Friend ReadOnly Property SleepTimerOnPostsLimit As PropertyValue
<Provider(NameOf(SleepTimerOnPostsLimit), FieldsChecker:=True)>
Private ReadOnly Property SleepTimerOnPostsLimitProvider As IFormatProvider
@@ -161,6 +205,21 @@ Namespace API.Instagram
#Region "429 bypass"
<PXML("InstagramDownloadingErrorDate")>
Private ReadOnly Property DownloadingErrorDate As PropertyValue
<Provider(NameOf(DownloadingErrorDate))>
Private ReadOnly Property DownloadingErrorDateProvider As IFormatProvider =
New CustomProvider(Function(ByVal v As Object, ByVal d As Type) As Object
If d Is GetType(Date) Then
Return AConvert(Of Date)(v, AModes.Var, Nothing)
ElseIf d Is GetType(String) Then
If Not IsNothing(v) AndAlso TypeOf v Is Date AndAlso CDate(v) = Date.MinValue Then
Return String.Empty
Else
Return AConvert(Of String)(v, AModes.XML, String.Empty)
End If
Else
Return Nothing
End If
End Function)
Friend Property LastApplyingValue As Integer? = Nothing
Friend ReadOnly Property ReadyForDownload As Boolean
Get
@@ -174,11 +233,64 @@ Namespace API.Instagram
End With
End Get
End Property
Private Const LastDownloadDateResetInterval As Integer = 60
<PXML> Private ReadOnly Property LastDownloadDate As PropertyValue
<PXML> Private ReadOnly Property LastRequestsCount As PropertyValue
Private ReadOnly MyLastRequests As Dictionary(Of Date, Integer)
Private ReadOnly Property MyLastRequestsDate As Date
Get
Try
Return If(MyLastRequests.Count > 0, MyLastRequests.Keys.Max, Now.AddDays(-1))
Catch ex As Exception
Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, "[SiteSettings.Instagram.MyLastRequestsDate]", Now.AddDays(-1))
End Try
End Get
End Property
Private Property MyLastRequestsCount As Integer
Get
Try
Return If(MyLastRequests.Count > 0, MyLastRequests.Values.Sum, 0)
Catch ex As Exception
Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, "[SiteSettings.Instagram.MyLastRequestsCount]", 0)
End Try
End Get
Set(ByVal NewValue As Integer)
If Not MyLastRequests.ContainsKey(ActiveSessionDate) Then
MyLastRequests.Add(ActiveSessionDate, NewValue)
Else
MyLastRequests(ActiveSessionDate) += NewValue
End If
End Set
End Property
Private Sub RefreshMyLastRequests(Optional ByVal DateToReplace As Date? = Nothing)
Try
With MyLastRequests
If .Count > 0 Then
Dim d As Date
For i% = .Count - 1 To 0 Step -1
d = .Keys(i)
If (Not DateToReplace.HasValue OrElse ActiveJobs < 1 OrElse d <> ActiveSessionDate) And
d.AddMinutes(LastDownloadDateResetInterval) < Now Then .Remove(d)
Next
End If
If .Count > 0 Then
If DateToReplace.HasValue Then
If .Keys.Contains(ActiveSessionDate) Then
Dim v% = .Item(ActiveSessionDate)
.Remove(ActiveSessionDate)
.Add(DateToReplace.Value, v)
End If
End If
LastDownloadDate.Value = .Keys.Max
LastRequestsCount.Value = .Values.Sum
End If
End With
Catch ex As Exception
ErrorsDescriber.Execute(EDP.SendToLog, ex, "[SiteSettings.Instagram.RefreshMyLastRequests]")
End Try
End Sub
<PropertyOption(IsInformationLabel:=True), ControlNumber(100)>
Private Property LastRequestsCountLabel As PropertyValue
Private ReadOnly LastRequestsCountLabelStr As Func(Of Integer, String) = Function(r) $"Number of spent requests: {r.NumToGroupIntegral}"
Private ReadOnly Property LastRequestsCountLabel As PropertyValue
Private TooManyRequestsReadyForCatch As Boolean = True
Friend Function GetWaitDate() As Date
With DownloadingErrorDate
@@ -235,11 +347,14 @@ Namespace API.Instagram
browserExt = .Value(Header_BrowserExt)
platform = .Value(Header_Platform_Verion)
End If
'.Add(Header_IG_WWW_CLAIM, 0)
.Add("Dnt", 1)
.Add("Dpr", 1)
.Add("Sec-Ch-Ua-Mobile", "?0")
.Add("Sec-Ch-Ua-Model", """""")
.Add("Sec-Ch-Ua-Platform", """Windows""")
.Add("Sec-Fetch-Dest", "empty")
.Add("Sec-Fetch-Mode", "cors")
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, "empty"))
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "cors"))
.Add("Sec-Fetch-Site", "same-origin")
.Add("X-Requested-With", "XMLHttpRequest")
End With
@@ -257,6 +372,15 @@ Namespace API.Instagram
HH_PLATFORM = New PropertyValue(platform, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_PLATFORM), v))
HH_USER_AGENT = New PropertyValue(useragent, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_USER_AGENT), v))
HH_IG_WWW_CLAIM_UPDATE_INTERVAL = New PropertyValue(120)
HH_IG_WWW_CLAIM_ALWAYS_ZERO = New PropertyValue(False)
HH_IG_WWW_CLAIM_RESET_EACH_SESSION = New PropertyValue(True)
HH_IG_WWW_CLAIM_RESET_EACH_TARGET = New PropertyValue(True)
HH_IG_WWW_CLAIM_USE = New PropertyValue(True)
HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO = New PropertyValue(True)
TokenUpdateIntervalProvider = New TokenRefreshIntervalProvider
USE_GQL = New PropertyValue(False)
DownloadTimeline = New PropertyValue(True)
DownloadTimeline_Def = New PropertyValue(DownloadTimeline.Value, GetType(Boolean))
DownloadReels = New PropertyValue(False)
@@ -268,6 +392,8 @@ Namespace API.Instagram
DownloadTagged = New PropertyValue(False)
DownloadTagged_Def = New PropertyValue(DownloadTagged.Value, GetType(Boolean))
RequestsWaitTimer_Any = New PropertyValue(1000)
RequestsWaitTimer_AnyProvider = New TimersChecker(0)
RequestsWaitTimer = New PropertyValue(1000)
RequestsWaitTimerProvider = New TimersChecker(100)
RequestsWaitTimerTaskCount = New PropertyValue(1)
@@ -283,17 +409,22 @@ Namespace API.Instagram
TaggedNotifyLimit = New PropertyValue(200)
TaggedNotifyLimitProvider = New TaggedNotifyLimitChecker
DownloadingErrorDate = New PropertyValue(Nothing, GetType(Date))
DownloadingErrorDate = New PropertyValue(Now.AddYears(10), GetType(Date))
LastDownloadDate = New PropertyValue(Now.AddDays(-1))
LastRequestsCount = New PropertyValue(0)
LastRequestsCountLabel = New PropertyValue(LastRequestsCountLabelStr.Invoke(LastRequestsCount.Value))
LastRequestsCount.OnChangeFunction = Sub(vv) LastRequestsCountLabel.Value = LastRequestsCountLabelStr.Invoke(vv)
LastRequestsCountLabel = New PropertyValue(String.Empty, GetType(String))
MyLastRequests = New Dictionary(Of Date, Integer)
_AllowUserAgentUpdate = False
UrlPatternUser = "https://www.instagram.com/{0}/"
UserRegex = RParams.DMS(String.Format(UserRegexDefaultPattern, "instagram.com/"), 1)
ImageVideoContains = "instagram.com"
End Sub
Friend Overrides Sub EndInit()
Try : MyLastRequests.Add(LastDownloadDate.Value, LastRequestsCount.Value) : Catch : End Try
If Not CBool(HH_IG_WWW_CLAIM_USE.Value) Then Responser.Headers.Remove(Header_IG_WWW_CLAIM)
MyBase.EndInit()
End Sub
#End Region
#Region "PropertiesDataChecker"
<PropertiesDataChecker({NameOf(TaggedNotifyLimit)})>
@@ -326,11 +457,23 @@ Namespace API.Instagram
Return ActiveJobs < 2 AndAlso Not SkipUntilNextSession AndAlso ReadyForDownload AndAlso BaseAuthExists() AndAlso DownloadTimeline.Value
End Function
Private ActiveJobs As Integer = 0
Private ActiveSessionDate As Date
Private _NextWNM As UserData.WNM = UserData.WNM.Notify
Private _NextTagged As Boolean = True
Friend Overrides Sub DownloadStarted(ByVal What As Download)
ActiveJobs += 1
If CDate(LastDownloadDate.Value).AddMinutes(120) < Now Or Not ACheck(HH_IG_WWW_CLAIM.Value) Then HH_IG_WWW_CLAIM.Value = "0"
If ActiveJobs = 1 Then ActiveSessionDate = Now
If Not HH_IG_WWW_CLAIM_IS_ZERO AndAlso
(
(CBool(HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO.Value) AndAlso MyLastRequestsDate.AddMinutes(HH_IG_WWW_CLAIM_UPDATE_INTERVAL.Value) < Now) Or
Not ACheck(HH_IG_WWW_CLAIM.Value) Or
(
Not (
CBool(HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO.Value) And
(CBool(HH_IG_WWW_CLAIM_RESET_EACH_SESSION.Value) Or CBool(HH_IG_WWW_CLAIM_ALWAYS_ZERO.Value))
)
)
) Then HH_IG_WWW_CLAIM.Value = "0"
End Sub
Friend Overrides Sub BeforeStartDownload(ByVal User As Object, ByVal What As Download)
With DirectCast(User, UserData)
@@ -338,10 +481,9 @@ Namespace API.Instagram
.WaitNotificationMode = _NextWNM
.TaggedCheckSession = _NextTagged
End If
If CDate(LastDownloadDate.Value).AddMinutes(60) > Now Then
.RequestsCount = LastRequestsCount.Value
If MyLastRequestsDate.AddMinutes(LastDownloadDateResetInterval) > Now Then
.RequestsCount = MyLastRequestsCount
Else
LastRequestsCount.Value = 0
.RequestsCount = 0
End If
End With
@@ -351,8 +493,7 @@ Namespace API.Instagram
_NextWNM = .WaitNotificationMode
If _NextWNM = UserData.WNM.SkipTemp Or _NextWNM = UserData.WNM.SkipCurrent Then _NextWNM = UserData.WNM.Notify
_NextTagged = .TaggedCheckSession
LastDownloadDate.Value = Now
LastRequestsCount.Value = .RequestsCount
MyLastRequestsCount = .RequestsCountSession
_FieldsChangerSuspended = True
HH_IG_WWW_CLAIM.Value = Responser.Headers.Value(Header_IG_WWW_CLAIM)
HH_CSRF_TOKEN.Value = Responser.Headers.Value(Header_CSRF_TOKEN)
@@ -362,7 +503,7 @@ Namespace API.Instagram
Friend Overrides Sub DownloadDone(ByVal What As Download)
_NextWNM = UserData.WNM.Notify
_NextTagged = True
LastDownloadDate.Value = Now
RefreshMyLastRequests(Now)
ActiveJobs -= 1
SkipUntilNextSession = False
End Sub
@@ -382,6 +523,11 @@ Namespace API.Instagram
Private __DownloadStoriesUser As Boolean = False
Private __DownloadTagged As Boolean = False
Friend Overrides Sub BeginEdit()
RefreshMyLastRequests()
Dim v% = MyLastRequestsCount
Dim d$ = String.Empty
If v > 0 Then d = $" ({MyLastRequestsDate.ToStringDate(DateTimeDefaultProvider)})"
LastRequestsCountLabel.Value = $"Number of spent requests: {v.NumToGroupIntegral}{d}"
____HH_CSRF_TOKEN = AConvert(Of String)(HH_CSRF_TOKEN.Value, String.Empty)
____HH_IG_APP_ID = AConvert(Of String)(HH_IG_APP_ID.Value, String.Empty)
____HH_ASBD_ID = AConvert(Of String)(HH_ASBD_ID.Value, String.Empty)
@@ -460,6 +606,12 @@ Namespace API.Instagram
Return ErrorsDescriber.Execute(EDP.SendToLog, ex, "Can't open user's post", String.Empty)
End Try
End Function
#End Region
#Region "IDisposable Support"
Protected Overrides Sub Dispose(ByVal disposing As Boolean)
If Not disposedValue And disposing And Not MyLastRequests Is Nothing Then MyLastRequests.Clear()
MyBase.Dispose(disposing)
End Sub
#End Region
End Class
End Namespace

333
SCrawler/API/Instagram/UserData.GQL.vb Normal file
View File

@@ -0,0 +1,333 @@
' Copyright (C) Andy https://github.com/AAndyProgram
' This program is free software: you can redistribute it and/or modify
' it under the terms of the GNU General Public License as published by
' the Free Software Foundation, either version 3 of the License, or
' (at your option) any later version.
'
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports System.Threading
Imports SCrawler.API.Base
Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.RegularExpressions
Imports PersonalUtilities.Tools.Web.Clients
Imports PersonalUtilities.Tools.Web.Documents.JSON
Namespace API.Instagram
Partial Friend Class UserData
#Region "Tokens"
Protected Property Token_dtsg As String = String.Empty
Protected ReadOnly Property Token_dtsg_Var As String
Get
Return If(Token_dtsg.IsEmptyString, String.Empty, SymbolsConverter.ASCII.EncodeSymbolsOnly(Token_dtsg))
End Get
End Property
Protected Property Token_lsd As String = String.Empty
Protected Sub ResetBaseTokens()
Token_dtsg = String.Empty
Token_lsd = String.Empty
End Sub
#End Region
#Region "Headers"
Friend Const GQL_HEADER_FB_FRINDLY_NAME As String = "x-fb-friendly-name"
Friend Const GQL_HEADER_FB_LSD As String = "x-fb-lsd"
#End Region
#Region "Data constants"
Private Const GQL_UserData_DocId As String = "7381344031985950"
Private Const GQL_UserData_FbFriendlyName As String = "PolarisProfilePageContentQuery"
Private Const GQL_Highlights_DocId As String = "8298007123561120"
Private Const GQL_Highlights_DocId_Second As String = "7559771384111300"
Private Const GQL_Highlights_FbFriendlyName As String = "PolarisProfileStoryHighlightsTrayContentQuery"
Private Const GQL_Highlights_FbFriendlyName_Second As String = "PolarisStoriesV3HighlightsPageQuery"
Private Const GQL_UserStories_DocId As String = "25231722019806941"
Private Const GQL_UserStories_FbFriendlyName As String = "PolarisStoriesV3ReelPageStandaloneQuery"
Private Const GQL_Timeline_DocId As String = "7268577773270422"
Private Const GQL_Timeline_FbFriendlyName As String = "PolarisProfilePostsQuery"
Private Const GQL_Timeline_DocId_Second As String = "7286316061475375"
Private Const GQL_Timeline_FbFriendlyName_Second As String = "PolarisProfilePostsTabContentQuery_connection"
Private Const GQL_Reels_DocId As String = "7191572580905225"
Private Const GQL_Reels_FbFriendlyName As String = "PolarisProfileReelsTabContentQuery"
Private Const GQL_Tagged_DocId As String = "7289408964443685"
Private Const GQL_Tagged_FbFriendlyName As String = "PolarisProfileTaggedTabContentQuery"
#End Region
#Region "Url & var constants"
Private Const GQL_URL_PATTERN_VARS As String = "doc_id={0}&lsd={1}&fb_dtsg={2}&fb_api_req_friendly_name={3}&variables={4}"
Private Const GQL_URL As String = "https://www.instagram.com/api/graphql"
Private Const GQL_URL_Q As String = "https://www.instagram.com/graphql/query"
#End Region
#Region "Download functions"
Protected Sub UpdateHeadersGQL(ByVal HeaderValue As String)
Responser.Headers.Add(GQL_HEADER_FB_FRINDLY_NAME, HeaderValue)
Responser.Headers.Add(GQL_HEADER_FB_LSD, Token_lsd)
End Sub
<Obsolete("Use 'GET' function: 'GetUserData'", False)>
Private Sub GetUserDataGQL(ByVal Token As CancellationToken)
Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_UserData_DocId, Token_lsd, Token_dtsg_Var, GQL_UserData_FbFriendlyName,
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""id"":""{ID}"",""relay_header"":false,""render_surface"":""PROFILE""" & "}"))
UpdateRequestNumber()
ChangeResponserMode(True)
UpdateHeadersGQL(GQL_UserData_FbFriendlyName)
Dim r$ = Responser.GetResponse(GQL_URL, vars)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r)
If j.ListExists Then
With j({"data", "user"})
If .ListExists Then
UserSiteName = .Value("full_name").IfNullOrEmpty(UserSiteName)
Dim f As New SFile With {.Path = DownloadContentDefault_GetRootDir(), .Name = "ProfilePicture", .Extension = "jpg"}
Dim pic$ = .Value({"hd_profile_pic_url_info"}, "url").IfNullOrEmpty(.Value("profile_pic_url"))
If Not pic.IsEmptyString Then GetWebFile(pic, f, EDP.ReturnValue)
UserDescriptionUpdate(.Value("biography"))
End If
End With
End If
End Using
End If
End Sub
Private Function GetTimelineGQL(ByVal Cursor As String, ByVal Token As CancellationToken) As String
Const none_cursor$ = "none"
Dim nextCursor$ = String.Empty, hasNextPage$ = String.Empty
Dim vars$
ThrowAny(Token)
UpdateRequestNumber()
ChangeResponserMode(True)
If Cursor.IsEmptyString Then
vars = "{""data"":{""count"":50,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""username"":""" &
NameTrue & """,""__relay_internal__pv__PolarisShareMenurelayprovider"":false}"
vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Timeline_DocId, Token_lsd, Token_dtsg_Var, GQL_Timeline_FbFriendlyName,
SymbolsConverter.ASCII.EncodeSymbolsOnly(vars))
UpdateHeadersGQL(GQL_Timeline_FbFriendlyName)
Else
vars = "{""after"":""" & Cursor & """,""before"":null,""data"":{""count"":50,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""first"":50,""last"":null,""username"":""" &
NameTrue & """,""__relay_internal__pv__PolarisShareMenurelayprovider"":false}"
vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Timeline_DocId_Second, Token_lsd, Token_dtsg_Var, GQL_Timeline_FbFriendlyName_Second,
SymbolsConverter.ASCII.EncodeSymbolsOnly(vars))
UpdateHeadersGQL(GQL_Timeline_FbFriendlyName_Second)
End If
DefaultParser_ElemNode = {"node"}
Dim r$ = Responser.GetResponse(GQL_URL, vars)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r)
If j.ListExists Then
With j({"data", "xdt_api__v1__feed__user_timeline_graphql_connection"})
If .ListExists Then
With .Item("page_info")
If .ListExists Then
nextCursor = .Value("end_cursor")
hasNextPage = .Value("has_next_page").FromXML(Of Boolean)(False)
End If
End With
With .Item("edges")
If .ListExists Then
If Not DefaultParser(.Self, Sections.Timeline, Token) Then Throw New ExitException
End If
End With
End If
End With
End If
End Using
End If
Return If(hasNextPage And (Not nextCursor.IsEmptyString AndAlso Not nextCursor.StringToLower = none_cursor), nextCursor, String.Empty)
End Function
Private Function GetHighlightsGQL_List() As List(Of String)
Dim nextCursor$ = String.Empty, hasNextPage$ = String.Empty
Dim i% = -1
Dim hList As New List(Of String)
Dim tmpList As New List(Of String)
Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_Highlights_DocId, Token_lsd, Token_dtsg_Var, GQL_Highlights_FbFriendlyName,
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""user_id"":""{ID}""" & "}"))
UpdateRequestNumber()
ChangeResponserMode(True)
UpdateHeadersGQL(GQL_Highlights_FbFriendlyName)
Dim r$ = Responser.GetResponse(GQL_URL_Q, vars)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r)
If j.ListExists Then
'With j({"data"})
With j({"data", "highlights"})
If .ListExists Then
With .Item("page_info")
If .ListExists Then
nextCursor = .Value("end_cursor")
hasNextPage = .Value("has_next_page").FromXML(Of Boolean)(False)
End If
End With
With .Item({"edges"})
If .ListExists Then hList.ListAddList(.Select(Function(jj) jj.Value({"node"}, "id")), LNC)
End With
End If
End With
End If
End Using
End If
Return hList
End Function
Private Sub GetHighlightsGQL(ByRef StoriesList As List(Of String), ByVal Token As CancellationToken)
Const highlightData$ = """first"":50,""initial_reel_id"":""{0}"",""last"":2,""reel_ids"":[{1}]"
Dim tmpList As New List(Of String)
Dim i% = -1
If StoriesList.ListExists Then
tmpList.AddRange(StoriesList.Take(10))
StoriesList.RemoveRange(0, tmpList.Count)
Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_Highlights_DocId_Second, Token_lsd, Token_dtsg_Var, GQL_Highlights_FbFriendlyName_Second,
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(highlightData, tmpList(0), tmpList.Select(Function(hl) $"""{hl}""").ListToString(",")) & "}"))
ThrowAny(Token)
UpdateRequestNumber()
ChangeResponserMode(True)
UpdateHeadersGQL(GQL_Highlights_FbFriendlyName_Second)
Dim r$ = Responser.GetResponse(GQL_URL_Q, vars)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r)
If j.ListExists Then
With j({"data", "xdt_api__v1__feed__reels_media__connection", "edges"})
If .ListExists Then
ProgressPre.ChangeMax(.Count)
For Each n As EContainer In .Self : GetStoriesData_ParseSingleHighlight(n("node"), i, False, Token) : Next
End If
End With
End If
End Using
End If
tmpList.Clear()
End If
tmpList.Clear()
End Sub
Private Sub GetUserStoriesGQL(ByVal Token As CancellationToken)
'"{" & $"""user_id"":""{ID}""" & "}"
Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_UserStories_DocId, Token_lsd, Token_dtsg_Var, GQL_UserStories_FbFriendlyName,
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""reel_ids_arr"":[""{ID}""]" & "}"))
UpdateRequestNumber()
ChangeResponserMode(True)
UpdateHeadersGQL(GQL_UserStories_FbFriendlyName)
Dim r$ = Responser.GetResponse(GQL_URL, vars)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r)
If j.ListExists Then
Dim i% = -1
GetStoriesData_ParseSingleHighlight(j.ItemF({"data", "xdt_api__v1__feed__reels_media", "reels_media", 0}), i, True, Token)
End If
End Using
End If
End Sub
Private WriteOnly Property GetReelsGQL_SetEnvir As Boolean
Set(ByVal init As Boolean)
If init Then
ObtainMedia_SetReelsFunc()
DefaultParser_PostUrlCreator = Function(post) $"{MySiteSettings.GetUserUrl(Me).TrimEnd("/")}/reel/{post.Code}"
Else
ObtainMedia_SizeFuncPic = Nothing
ObtainMedia_SizeFuncVid = Nothing
DefaultParser_PostUrlCreator = DefaultParser_PostUrlCreator_Default
End If
End Set
End Property
''' <returns>Response</returns>
Private Function GetReelsGQL(ByVal Cursor As String) As String
GetReelsGQL_SetEnvir = True
Dim errData$ = String.Empty
If Cursor.IsEmptyString And Not ValidateBaseTokens() Then GetPageTokens()
If Cursor.IsEmptyString And Not ValidateBaseTokens(errData) Then ValidateBaseTokens_Error(errData)
Dim vars$ = """data"":{""include_feed_video"":true,""page_size"":50,""target_user_id"":""" & ID & """}"
If Not Cursor.IsEmptyString Then vars = $"""after"":""{Cursor}"",""before"":null,{vars},""first"":4,""last"":null"
vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Reels_DocId, Token_lsd, Token_dtsg_Var, GQL_Reels_FbFriendlyName,
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & vars & "}"))
UpdateRequestNumber()
ChangeResponserMode(True)
UpdateHeadersGQL(GQL_Reels_FbFriendlyName)
Return Responser.GetResponse(GQL_URL, vars)
End Function
''' <summary>Response</summary>
Private Function GetTaggedGQL(ByVal Cursor As String) As String
'default count = 12
'max count = 21
Dim vars$
If Cursor.IsEmptyString Then
vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Tagged_DocId, Token_lsd, Token_dtsg_Var, GQL_Tagged_FbFriendlyName,
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""count"":50,""user_id"":""{ID}""" & "}"))
Else
vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Tagged_DocId, Token_lsd, Token_dtsg_Var, GQL_Tagged_FbFriendlyName,
SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""after"":""{Cursor}"",""before"":null,""count"":50,""first"":50,""last"":null,""user_id"":""{ID}""" & "}"))
End If
UpdateRequestNumber()
ChangeResponserMode(True)
UpdateHeadersGQL(GQL_Tagged_FbFriendlyName)
Return Responser.GetResponse(GQL_URL, vars)
End Function
#End Region
#Region "ValidateBaseTokens"
Protected Overridable Overloads Function ValidateBaseTokens() As Boolean
Return ValidateBaseTokens(Nothing)
End Function
Protected Overridable Overloads Function ValidateBaseTokens(ByRef ErrData As String) As Boolean
ErrData = String.Empty
If Token_dtsg.IsEmptyString Then ErrData.StringAppend("dtsg")
If Token_lsd.IsEmptyString Then ErrData.StringAppend("lsd")
Return ErrData.IsEmptyString
End Function
Protected Overridable Sub ValidateBaseTokens_Error(Optional ByVal ErrData As String = "")
If _UseGQL Then DisableSection(Sections.Timeline)
ExitException.ThrowTokens(Me, ErrData)
End Sub
#End Region
#Region "GetPageTokens"
Private Sub GetPageTokens()
ResetBaseTokens()
Try
UpdateRequestNumber()
ChangeResponserMode(False, Not _UseGQL)
With Responser
With .Headers
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, "document"))
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "navigate"))
End With
End With
Dim r$ = Responser.GetResponse(MySiteSettings.GetUserUrl(Me))
If Not r.IsEmptyString Then
Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)
Dim tokens As List(Of String) = RegexReplace(r, rr)
Dim tt$, ttVal$
If tokens.ListExists Then
With rr
.Match = Nothing
.MatchSub = 1
.WhatGet = RegexReturn.Value
End With
For Each tt In tokens
If Not Token_lsd.IsEmptyString And Not Token_dtsg.IsEmptyString Then
Exit For
Else
ttVal = RegexReplace(tt, rr)
If Not ttVal.IsEmptyString Then
If ttVal.Contains(":") Then
If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal
Else
If Token_lsd.IsEmptyString Then Token_lsd = ttVal
End If
End If
End If
Next
End If
End If
Catch ex As Exception
Finally
ChangeResponserMode(_UseGQL, Not _UseGQL)
End Try
End Sub
#End Region
End Class
End Namespace

459
SCrawler/API/Instagram/UserData.vb
View File

@@ -69,7 +69,6 @@ Namespace API.Instagram
Return New EContainer("Post", ID, {New EAttribute(Name_Section, CInt(Section)), New EAttribute(Name_Code, Code)})
End Function
End Structure
Friend Const Header_FB_LSD As String = "x-fb-lsd"
Private ReadOnly Property MySiteSettings As SiteSettings
Get
Return DirectCast(HOST.Source, SiteSettings)
@@ -139,14 +138,19 @@ Namespace API.Instagram
Friend Sub New()
PostsKVIDs = New List(Of PostKV)
PostsToReparse = New List(Of PostKV)
_ResponserAutoUpdateCookies = True
End Sub
#End Region
#Region "Download data"
Private WwwClaimUpdate As Boolean = True
Private WwwClaimUpdate_R As Boolean = True
Private WwwClaimDefaultAlgo As Boolean = True
Private WwwClaimUse As Boolean = True
Private E560Thrown As Boolean = False
Friend Err5xx As Integer = -1
Private Class ExitException : Inherits Exception
Friend Property Is560 As Boolean = False
Friend Property IsTokens As Boolean = False
Friend Property TokensData As String = String.Empty
Friend Shared Sub Throw560(ByRef Source As UserData)
If Not Source.E560Thrown Then
MyMainLOG = $"{Source.ToStringForLog}: ({IIf(Source.Err5xx > 0, Source.Err5xx, 560)}) Download skipped until next session"
@@ -154,6 +158,10 @@ Namespace API.Instagram
End If
Throw New ExitException With {.Is560 = True}
End Sub
Friend Shared Sub ThrowTokens(ByRef Source As UserData, ByVal Data As String)
MyMainLOG = $"{Source.ToStringForLog}: failed to update some{IIf(Data.IsEmptyString, String.Empty, $" ({Data})")} credentials"
Throw New ExitException With {.IsTokens = True, .TokensData = Data}
End Sub
End Class
Private ReadOnly Property MyFilePostsKV As SFile
Get
@@ -235,8 +243,75 @@ Namespace API.Instagram
Private _DownloadingInProgress As Boolean = False
Private _Limit As Integer = -1
Private _TotalPostsParsed As Integer = 0
Private _LastWwwClaim As String = String.Empty
Private _ResponserGQLMode As Boolean = False
Private _UseGQL As Boolean = False
Private Sub ChangeResponserMode(ByVal GQL As Boolean, Optional ByVal Force As Boolean = False)
If Not _ResponserGQLMode = GQL Or Force Then
_ResponserGQLMode = GQL
ChangeResponserMode_StoreWwwClaim()
Responser.Headers.Clear()
Responser.Headers.AddRange(MySiteSettings.Responser.Headers)
If GQL Then
WwwClaimUpdate = False
With Responser
.Method = "POST"
.ContentType = "application/x-www-form-urlencoded"
.Referer = MySiteSettings.GetUserUrl(Me)
.CookiesExtractMode = Responser.CookiesExtractModes.Any
With .Headers
.Remove(SiteSettings.Header_IG_WWW_CLAIM)
.Add("origin", "https://www.instagram.com")
.Add("authority", "www.instagram.com")
End With
End With
Else
WwwClaimUpdate = WwwClaimUpdate_R
With Responser
.Method = "GET"
.ContentType = Nothing
.Referer = Nothing
.CookiesExtractMode = MySiteSettings.Responser.CookiesExtractMode
With .Headers
.Remove("origin")
.Remove("authority")
.Remove(GQL_HEADER_FB_FRINDLY_NAME)
.Remove(GQL_HEADER_FB_LSD)
Dim hv$ = MySiteSettings.Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest)).IfNullOrEmpty("empty")
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, hv))
hv = MySiteSettings.Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode)).IfNullOrEmpty("cors")
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, hv))
If Not _UseGQL And WwwClaimUse Then .Add(SiteSettings.Header_IG_WWW_CLAIM, _LastWwwClaim)
End With
End With
End If
End If
End Sub
Private Sub ChangeResponserMode_StoreWwwClaim()
If Not _UseGQL Then
With Responser.Headers
If .Contains(SiteSettings.Header_IG_WWW_CLAIM) AndAlso Not .Value(SiteSettings.Header_IG_WWW_CLAIM).IsEmptyString Then _LastWwwClaim = .Value(SiteSettings.Header_IG_WWW_CLAIM)
End With
End If
End Sub
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
ResetBaseTokens()
UserNameRequested = False
RequestsCountSession = 0
_LastWwwClaim = String.Empty
_ResponserGQLMode = False
_UseGQL = MySiteSettings.USE_GQL.Value
WwwClaimUse = MySiteSettings.HH_IG_WWW_CLAIM_USE.Value
WwwClaimDefaultAlgo = MySiteSettings.HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO.Value
With MySiteSettings : WwwClaimUpdate = (Not CBool(.HH_IG_WWW_CLAIM_ALWAYS_ZERO.Value) And CBool(.HH_IG_WWW_CLAIM_USE.Value)) Or
WwwClaimDefaultAlgo : End With
WwwClaimUpdate_R = WwwClaimUpdate
Dim upClaimRequest As Action = Sub() If WwwClaimUpdate And Not WwwClaimDefaultAlgo And CBool(MySiteSettings.HH_IG_WWW_CLAIM_RESET_EACH_TARGET.Value) Then _
Responser.Headers.Add(SiteSettings.Header_IG_WWW_CLAIM, 0)
DefaultParser_ElemNode = Nothing
ChangeResponserMode(_UseGQL)
Dim s As Sections = Sections.Timeline
Dim errorFound As Boolean = False
Try
@@ -251,6 +326,7 @@ Namespace API.Instagram
Dim dt As Func(Of Boolean) = Function() (CBool(MySiteSettings.DownloadTimeline.Value) And GetTimeline) Or IsSavedPosts
If dt.Invoke And Not LastCursor.IsEmptyString Then
s = IIf(IsSavedPosts, Sections.SavedPosts, Sections.Timeline)
upClaimRequest.Invoke
DownloadData(LastCursor, s, Token)
ProgressPre.Done()
ThrowAny(Token)
@@ -258,27 +334,51 @@ Namespace API.Instagram
End If
If dt.Invoke And Not HasError Then
s = IIf(IsSavedPosts, Sections.SavedPosts, Sections.Timeline)
upClaimRequest.Invoke
ChangeResponserMode(_UseGQL)
DownloadData(String.Empty, s, Token)
ProgressPre.Done()
ThrowAny(Token)
If Not HasError Then FirstLoadingDone = True
End If
DefaultParser_ElemNode = Nothing
If FirstLoadingDone Then LastCursor = String.Empty
If Not IsSavedPosts AndAlso MySiteSettings.BaseAuthExists() Then
DefaultParser_ElemNode = Nothing
ChangeResponserMode(_UseGQL)
If CBool(MySiteSettings.DownloadReels.Value) And GetReels Then
s = Sections.Reels
DefaultParser_ElemNode = {"node", "media"}
upClaimRequest.Invoke
ChangeResponserMode(True)
DownloadData(String.Empty, s, Token)
DefaultParser_ElemNode = Nothing
DownloadReels_SetEnvir = False
GetReelsGQL_SetEnvir = False
ProgressPre.Done()
End If
If CBool(MySiteSettings.DownloadStories.Value) And GetStories Then s = Sections.Stories : DownloadData(String.Empty, s, Token) : ProgressPre.Done()
If CBool(MySiteSettings.DownloadStoriesUser.Value) And GetStoriesUser Then s = Sections.UserStories : DownloadData(String.Empty, s, Token) : ProgressPre.Done()
If CBool(MySiteSettings.DownloadTagged.Value) And GetTaggedData Then
s = Sections.Tagged
DefaultParser_ElemNode = Nothing
ChangeResponserMode(_UseGQL)
If CBool(MySiteSettings.DownloadStories.Value) And GetStories Then
s = Sections.Stories
upClaimRequest.Invoke
DownloadData(String.Empty, s, Token)
ProgressPre.Done()
End If
DefaultParser_ElemNode = Nothing
ChangeResponserMode(_UseGQL)
If CBool(MySiteSettings.DownloadStoriesUser.Value) And GetStoriesUser Then
s = Sections.UserStories
upClaimRequest.Invoke
DownloadData(String.Empty, s, Token)
ProgressPre.Done()
End If
DefaultParser_ElemNode = Nothing
ChangeResponserMode(_UseGQL)
If CBool(MySiteSettings.DownloadTagged.Value) And GetTaggedData Then
s = Sections.Tagged
upClaimRequest.Invoke
DownloadData(String.Empty, s, Token)
ProgressPre.Done()
DefaultParser_ElemNode = Nothing
If PostsToReparse.Count > 0 Then DownloadPosts(Token, True)
End If
End If
@@ -289,7 +389,7 @@ Namespace API.Instagram
Throw ex
Finally
DefaultParser_ElemNode = Nothing
DownloadReels_SetEnvir = False
GetReelsGQL_SetEnvir = False
E560Thrown = False
UpdateResponser()
ValidateExtension()
@@ -315,13 +415,13 @@ Namespace API.Instagram
If _DownloadingInProgress AndAlso Not Responser Is Nothing AndAlso Not Responser.Disposed Then
_DownloadingInProgress = False
Responser_ResponseReceived_RemoveHandler()
Declarations.UpdateResponser(Responser, MySiteSettings.Responser)
Declarations.UpdateResponser(Responser, MySiteSettings.Responser, WwwClaimUpdate)
End If
Catch
End Try
End Sub
Protected Overrides Sub Responser_ResponseReceived(ByVal Sender As Object, ByVal e As EventArguments.WebDataResponse)
Declarations.UpdateResponser(e, Responser)
Declarations.UpdateResponser(e, Responser, WwwClaimUpdate)
End Sub
Protected Enum Sections : Timeline : Reels : Tagged : Stories : UserStories : SavedPosts : End Enum
Protected Const StoriesFolder As String = "Stories"
@@ -329,6 +429,12 @@ Namespace API.Instagram
#Region "429 bypass"
Private Const MaxPostsCount As Integer = 200
Friend Property RequestsCount As Integer = 0
Friend Property RequestsCountSession As Integer = 0
Private Sub UpdateRequestNumber()
If CInt(MySiteSettings.RequestsWaitTimer_Any.Value) > 0 Then Thread.Sleep(CInt(MySiteSettings.RequestsWaitTimer_Any.Value))
RequestsCount += 1
RequestsCountSession += 1
End Sub
Friend Enum WNM As Integer
Notify = 0
SkipCurrent = 1
@@ -468,46 +574,74 @@ Namespace API.Instagram
Dim HasNextPage As Boolean = False
Dim EndCursor$ = String.Empty
Dim PostID$ = String.Empty, PostDate$ = String.Empty, SpecFolder$ = String.Empty
Dim TokensErrData$ = String.Empty
Dim PostIDKV As PostKV
Dim ENode() As Object = Nothing
Dim processGetResponse As Boolean = True
NextRequest(True)
'Check environment
If Not IsSavedPosts Then
If ID.IsEmptyString Then GetUserId()
If ID.IsEmptyString Then GetUserData()
If ID.IsEmptyString Then Throw New Plugin.ExitException("can't get user ID")
If _UseGQL And Cursor.IsEmptyString And Not Section = Sections.SavedPosts Then
If Not ValidateBaseTokens() Then GetPageTokens()
If Not ValidateBaseTokens(TokensErrData) Then ValidateBaseTokens_Error(TokensErrData)
End If
End If
'Create query
Select Case Section
Case Sections.Timeline
URL = $"https://www.instagram.com/api/v1/feed/user/{NameTrue}/username/?count=50" &
If(Cursor.IsEmptyString, String.Empty, $"&max_id={Cursor}")
ENode = Nothing
If _UseGQL Then
EndCursor = GetTimelineGQL(Cursor, Token)
HasNextPage = Not EndCursor.IsEmptyString
MySiteSettings.TooManyRequests(False)
GoTo NextPageBlock
Else
URL = $"https://www.instagram.com/api/v1/feed/user/{NameTrue}/username/?count=50" &
If(Cursor.IsEmptyString, String.Empty, $"&max_id={Cursor}")
ENode = Nothing
End If
Case Sections.Reels
r = DownloadReels(Cursor, Token)
ChangeResponserMode(True)
r = GetReelsGQL(Cursor)
ENode = {"data", "xdt_api__v1__clips__user__connection_v2"}
processGetResponse = False
Case Sections.SavedPosts
SavedPostsDownload(String.Empty, Token)
Exit Sub
ChangeResponserMode(False)
EndCursor = SavedPostsDownload(String.Empty, Token)
HasNextPage = Not EndCursor.IsEmptyString
MySiteSettings.TooManyRequests(False)
ThrowAny(Token)
GoTo NextPageBlock
Case Sections.Tagged
Dim vars$ = "{""id"":" & ID & ",""first"":50,""after"":""" & Cursor & """}"
vars = SymbolsConverter.ASCII.EncodeSymbolsOnly(vars)
URL = $"https://www.instagram.com/graphql/query/?doc_id=17946422347485809&variables={vars}"
ENode = {"data", "user", "edge_user_to_photos_of_you"}
SpecFolder = TaggedFolder
If _UseGQL Then
r = GetTaggedGQL(Cursor)
ENode = {"data", "xdt_api__v1__usertags__user_id__feed_connection"}
processGetResponse = False
Else
Dim vars$ = "{""id"":" & ID & ",""first"":50,""after"":""" & Cursor & """}"
vars = SymbolsConverter.ASCII.EncodeSymbolsOnly(vars)
URL = $"https://www.instagram.com/graphql/query/?doc_id=17946422347485809&variables={vars}"
ENode = {"data", "user", "edge_user_to_photos_of_you"}
End If
Case Sections.Stories
If Not StoriesRequested Then
StoriesList = GetStoriesList()
StoriesList = If(_UseGQL, GetHighlightsGQL_List(), GetStoriesList())
StoriesRequested = True
MySiteSettings.TooManyRequests(False)
RequestsCount += 1
ThrowAny(Token)
Continue Do
End If
If StoriesList.ListExists Then
GetStoriesData(StoriesList, False, Token)
If _UseGQL Then
GetHighlightsGQL(StoriesList, Token)
Else
GetStoriesData(StoriesList, False, Token)
End If
MySiteSettings.TooManyRequests(False)
RequestsCount += 1
End If
If StoriesList.ListExists Then
Continue Do
@@ -515,16 +649,17 @@ Namespace API.Instagram
Throw New ExitException
End If
Case Sections.UserStories
GetStoriesData(Nothing, True, Token)
If _UseGQL Then GetUserStoriesGQL(Token) Else GetStoriesData(Nothing, True, Token)
MySiteSettings.TooManyRequests(False)
RequestsCount += 1
Throw New ExitException
End Select
'Get response
If Not Section = Sections.Reels Then r = Responser.GetResponse(URL,, EDP.ThrowException)
If processGetResponse Then
UpdateRequestNumber()
r = Responser.GetResponse(URL)
End If
MySiteSettings.TooManyRequests(False)
RequestsCount += 1
ThrowAny(Token)
'Parsing
@@ -608,6 +743,7 @@ Namespace API.Instagram
Else
Throw New ExitException
End If
NextPageBlock:
dValue = 0
If HasNextPage And Not EndCursor.IsEmptyString Then DownloadData(EndCursor, Section, Token)
Catch jsonNull As JsonDocumentException When jsonNull.State = WebDocumentEventArgs.States.Error And
@@ -625,6 +761,8 @@ Namespace API.Instagram
Catch eex2 As ExitException
If eex2.Is560 Then
Throw New Plugin.ExitException With {.Silent = True}
ElseIf eex2.IsTokens And _UseGQL Then
Throw New Plugin.ExitException With {.Silent = True}
Else
If Not Section = Sections.Reels And (Section = Sections.Timeline Or Section = Sections.Tagged) And Not Cursor.IsEmptyString Then Throw eex2
End If
@@ -641,6 +779,7 @@ Namespace API.Instagram
Dim before%
Dim specFolder$ = IIf(IsTagged, "Tagged", String.Empty)
If PostsToReparse.Count > 0 Then ProgressPre.ChangeMax(PostsToReparse.Count)
ChangeResponserMode(False)
Try
Do While dValue = 1
ThrowAny(Token)
@@ -660,9 +799,9 @@ Namespace API.Instagram
ThrowAny(Token)
NextRequest(((i + 1) Mod 5) = 0)
ThrowAny(Token)
UpdateRequestNumber()
r = Responser.GetResponse(URL,, e)
MySiteSettings.TooManyRequests(False)
RequestsCount += 1
If Not r.IsEmptyString Then
j = JsonDocument.Parse(r)
If Not j Is Nothing Then
@@ -695,27 +834,30 @@ Namespace API.Instagram
ProcessException(DoEx, Token, $"downloading posts error [{URL}]",, Sections.Tagged)
End Try
End Sub
Private Sub SavedPostsDownload(ByVal Cursor As String, ByVal Token As CancellationToken)
''' <summary>Cursor</summary>
Private Function SavedPostsDownload(ByVal Cursor As String, ByVal Token As CancellationToken) As String
Dim URL$ = $"https://www.instagram.com/api/v1/feed/saved/posts/?max_id={Cursor}"
Dim HasNextPage As Boolean = False
Dim NextCursor$ = String.Empty
ThrowAny(Token)
Dim processNext As Boolean = False
UpdateRequestNumber()
Dim r$ = Responser.GetResponse(URL)
Dim nodes As IEnumerable(Of EContainer) = Nothing
If Not r.IsEmptyString Then
Using e As EContainer = JsonDocument.Parse(r)
If If(e?.Count, 0) > 0 Then
If e.ListExists Then
With e
HasNextPage = .Value("more_available").FromXML(Of Boolean)(False)
NextCursor = .Value("next_max_id")
If .Contains("items") Then nodes = (From ee As EContainer In .Item("items") Where ee.Count > 0 Select ee(0))
End With
If nodes.ListExists AndAlso DefaultParser(nodes, Sections.SavedPosts, Token) AndAlso
HasNextPage AndAlso Not NextCursor.IsEmptyString Then SavedPostsDownload(NextCursor, Token)
HasNextPage AndAlso Not NextCursor.IsEmptyString Then processNext = True
End If
End Using
End If
End Sub
Return If(processNext, NextCursor, String.Empty)
End Function
Protected DefaultParser_ElemNode() As Object = Nothing
Protected DefaultParser_IgnorePass As Boolean = False
Private ReadOnly DefaultParser_PostUrlCreator_Default As Func(Of PostKV, String) = Function(post) $"https://www.instagram.com/p/{post.Code}/"
@@ -740,25 +882,29 @@ Namespace API.Instagram
For Each nn In Items
ProgressPre.Perform()
With If(Not DefaultParser_ElemNode Is Nothing, nn.ItemF(DefaultParser_ElemNode), nn)
PostIDKV = New PostKV(.Value("code"), .Value("id"), Section)
PostOriginUrl = DefaultParser_PostUrlCreator(PostIDKV)
Pinned = .Contains("timeline_pinned_user_ids")
If Not DefaultParser_IgnorePass AndAlso PostKvExists(PostIDKV) Then
If Not Pinned Then Return False
Else
_TempPostsList.Add(PostIDKV.ID)
PostsKVIDs.ListAddValue(PostIDKV, LNC)
PostDate = .Value("taken_at")
If Not DefaultParser_IgnorePass And Not IsSavedPosts Then
Select Case CheckDatesLimit(PostDate, UnixDate32Provider)
Case DateResult.Skip : Continue For
Case DateResult.Exit : If Not Pinned Then Return False
End Select
If .ListExists Then
PostIDKV = New PostKV(.Value("code"), .Value("id"), Section)
PostOriginUrl = DefaultParser_PostUrlCreator(PostIDKV)
Pinned = .Contains("timeline_pinned_user_ids")
If (Section = Sections.Timeline And Not DefaultParser_IgnorePass) AndAlso PostKvExists(PostIDKV) Then
If Not Pinned Then Return False
Else
_TempPostsList.Add(PostIDKV.ID)
PostsKVIDs.ListAddValue(PostIDKV, LNC)
PostDate = .Value("taken_at")
If Not DefaultParser_IgnorePass And Not IsSavedPosts Then
Select Case CheckDatesLimit(PostDate, UnixDate32Provider)
Case DateResult.Skip : Continue For
Case DateResult.Exit : If Not Pinned Then Return False
End Select
End If
before = _TempMediaList.Count
ObtainMedia(.Self, PostIDKV.ID, SpecFolder, PostDate,, PostOriginUrl, State, Attempts)
If Not before = _TempMediaList.Count Then _TotalPostsParsed += 1
If _Limit > 0 And _TotalPostsParsed >= _Limit Then Return False
End If
before = _TempMediaList.Count
ObtainMedia(.Self, PostIDKV.ID, SpecFolder, PostDate,, PostOriginUrl, State, Attempts)
If Not before = _TempMediaList.Count Then _TotalPostsParsed += 1
If _Limit > 0 And _TotalPostsParsed >= _Limit Then Return False
Else
Return False
End If
End With
Next
@@ -768,106 +914,6 @@ Namespace API.Instagram
End If
End Function
#End Region
#Region "Get reels"
Private _GetReels_LSD As String = String.Empty
Private _GetReels_dtsg As String = String.Empty
Private ReadOnly Property DownloadReels_Tokens_Valid As Boolean
Get
Return Not _GetReels_LSD.IsEmptyString And Not _GetReels_dtsg.IsEmptyString
End Get
End Property
Private WriteOnly Property DownloadReels_SetEnvir As Boolean
Set(ByVal init As Boolean)
If init Then
ObtainMedia_SetReelsFunc()
DefaultParser_PostUrlCreator = Function(post) $"{MySiteSettings.GetUserUrl(Me).TrimEnd("/")}/reel/{post.Code}"
Else
ObtainMedia_SizeFuncPic = Nothing
ObtainMedia_SizeFuncVid = Nothing
DefaultParser_PostUrlCreator = DefaultParser_PostUrlCreator_Default
End If
End Set
End Property
Private Class Responser2 : Inherits Responser
Friend Sub New(ByVal Source As Responser)
MyBase.New
Copy(Source)
ErrorProcessor = New ResponserErrorProcessor(Source)
End Sub
End Class
''' <returns>Response</returns>
Private Function DownloadReels(ByVal Cursor As String, ByVal Token As CancellationToken) As String
Const requestPattern$ = "https://www.instagram.com/api/graphql?fb_dtsg={0}&fb_api_req_friendly_name=PolarisProfileReelsTabContentQuery&lsd={1}&doc_id=7191572580905225&variables={2}"
DownloadReels_SetEnvir = True
If Cursor.IsEmptyString And Not DownloadReels_Tokens_Valid Then GetPageTokens()
If Cursor.IsEmptyString And Not DownloadReels_Tokens_Valid Then Throw New ExitException
Using resp As New Responser2(Responser)
Try
resp.Method = "POST"
AddHandler resp.ResponseReceived, AddressOf Responser_ResponseReceived
resp.Headers.Add(Header_FB_LSD, _GetReels_LSD)
Dim vars$ = """data"":{""include_feed_video"":true,""page_size"":50,""target_user_id"":""" & ID & """}"
If Not Cursor.IsEmptyString Then vars = $"""after"":""{Cursor}"",""before"":null,{vars},""first"":4,""last"":null"
vars = "{" & vars & "}"
Dim url$ = String.Format(requestPattern, _GetReels_dtsg, _GetReels_LSD, SymbolsConverter.ASCII.EncodeSymbolsOnly(vars))
Return resp.GetResponse(url,, EDP.ThrowException)
Finally
With resp
Responser.Cookies.Update(.Cookies)
With .Headers
If .Contains(SiteSettings.Header_IG_WWW_CLAIM) Then Responser.Headers.Add(SiteSettings.Header_IG_WWW_CLAIM, .Value(SiteSettings.Header_IG_WWW_CLAIM))
If .Contains(SiteSettings.Header_CSRF_TOKEN) Then Responser.Headers.Add(SiteSettings.Header_CSRF_TOKEN, .Value(SiteSettings.Header_CSRF_TOKEN))
End With
End With
End Try
End Using
End Function
Private Function GetPageTokens() As Boolean
_GetReels_LSD = String.Empty
_GetReels_dtsg = String.Empty
Try
Dim r$ = Responser.GetResponse(MySiteSettings.GetUserUrl(Me),, EDP.ThrowException)
If Not r.IsEmptyString Then
Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)
Dim tokens As List(Of String) = RegexReplace(r, rr)
Dim tt$, ttVal$
If tokens.ListExists Then
With rr
.Match = Nothing
.MatchSub = 1
.WhatGet = RegexReturn.Value
End With
For Each tt In tokens
If Not _GetReels_LSD.IsEmptyString And Not _GetReels_dtsg.IsEmptyString Then
Exit For
Else
ttVal = RegexReplace(tt, rr)
If Not ttVal.IsEmptyString Then
If ttVal.Contains(":") Then
If _GetReels_dtsg.IsEmptyString Then _GetReels_dtsg = ttVal
Else
If _GetReels_LSD.IsEmptyString Then _GetReels_LSD = ttVal
End If
End If
End If
Next
End If
End If
Catch ex As Exception
Dim notFound$ = String.Empty
If _GetReels_dtsg.IsEmptyString Then notFound.StringAppend(Header_FB_LSD)
If _GetReels_LSD.IsEmptyString Then notFound.StringAppend("lsd")
LogError(ex, $"failed to update some{IIf(notFound.IsEmptyString, String.Empty, $" ({notFound})")} credentials", EDP.SendToLog)
End Try
Return DownloadReels_Tokens_Valid
End Function
#End Region
#Region "Code ID converters"
Protected Function CodeToID(ByVal Code As String) As String
Const CodeSymbols$ = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
@@ -1024,11 +1070,12 @@ Namespace API.Instagram
End Sub
#End Region
#Region "GetUserId, GetUserName"
Private Sub GetUserId()
Private Sub GetUserData()
Dim __idFound As Boolean = False
Try
RequestsCount += 1
Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/users/web_profile_info/?username={NameTrue}",, EDP.ThrowException)
ChangeResponserMode(False)
UpdateRequestNumber()
Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/users/web_profile_info/?username={NameTrue}")
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r)
If Not j Is Nothing AndAlso j.Contains({"data", "user"}) Then
@@ -1042,7 +1089,7 @@ Namespace API.Instagram
Dim eUrl$ = .Value("external_url")
If Not eUrl.IsEmptyString AndAlso (descr.IsEmptyString OrElse Not descr.Contains(eUrl)) Then descr.StringAppendLine(eUrl)
UserDescriptionUpdate(descr)
Dim f As New SFile With {.Path = MyFile.CutPath.Path, .Name = "ProfilePicture", .Extension = "jpg"}
Dim f As New SFile With {.Path = DownloadContentDefault_GetRootDir(), .Name = "ProfilePicture", .Extension = "jpg"}
If Not f.Exists Then
Dim profilePicture$ = .Value("profile_pic_url_hd")
If profilePicture.IsEmptyString OrElse Not GetWebFile(profilePicture, f, EDP.ReturnValue) Then
@@ -1062,13 +1109,15 @@ Namespace API.Instagram
LogError(ex, "get Instagram user ID")
End If
End If
Finally
ChangeResponserMode(_UseGQL)
End Try
End Sub
Private Function GetUserNameById() As Boolean
UserNameRequested = True
Try
If Not ID.IsEmptyString Then
RequestsCount += 1
UpdateRequestNumber()
Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/users/{ID}/info/",, EDP.ReturnValue)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue)
@@ -1101,9 +1150,9 @@ Namespace API.Instagram
Private Sub GetStoriesData(ByRef StoriesList As List(Of String), ByVal GetUserStory As Boolean, ByVal Token As CancellationToken)
Const ReqUrl$ = "https://i.instagram.com/api/v1/feed/reels_media/?{0}"
Dim tmpList As IEnumerable(Of String) = Nothing
Dim qStr$, r$, sFolder$, storyID$, pid$
Dim qStr$, r$
Dim i% = -1
Dim jj As EContainer, s As EContainer
Dim jj As EContainer
ThrowAny(Token)
If StoriesList.ListExists Or GetUserStory Then
If Not GetUserStory Then tmpList = StoriesList.Take(5)
@@ -1113,38 +1162,14 @@ Namespace API.Instagram
Else
qStr = String.Format(ReqUrl, tmpList.Select(Function(q) $"reel_ids=highlight:{q}").ListToString("&"))
End If
UpdateRequestNumber()
r = Responser.GetResponse(qStr,, EDP.ThrowException)
ThrowAny(Token)
If Not r.IsEmptyString Then
Using j As EContainer = JsonDocument.Parse(r).XmlIfNothing
If j.Contains("reels") Then
ProgressPre.ChangeMax(j("reels").Count)
For Each jj In j("reels")
ProgressPre.Perform()
i += 1
sFolder = jj.Value("title").StringRemoveWinForbiddenSymbols
storyID = jj.Value("id").Replace("highlight:", String.Empty)
If GetUserStory Then
sFolder = $"{StoriesFolder} (user)"
Else
If sFolder.IsEmptyString Then sFolder = $"Story_{storyID}"
If sFolder.IsEmptyString Then sFolder = $"Story_{i}"
sFolder = $"{StoriesFolder}\{sFolder}"
End If
If Not storyID.IsEmptyString Then storyID &= ":"
With jj("items").XmlIfNothing
If .Count > 0 Then
For Each s In .Self
pid = storyID & s.Value("id")
If Not _TempPostsList.Contains(pid) Then
ThrowAny(Token)
ObtainMedia(s, pid, sFolder)
_TempPostsList.Add(pid)
End If
Next
End If
End With
Next
For Each jj In j("reels") : GetStoriesData_ParseSingleHighlight(jj, i, GetUserStory, Token) : Next
End If
End Using
End If
@@ -1152,8 +1177,39 @@ Namespace API.Instagram
End If
End If
End Sub
Private Sub GetStoriesData_ParseSingleHighlight(ByVal Node As EContainer, ByRef Index As Integer, ByVal GetUserStory As Boolean, ByVal Token As CancellationToken)
If Not Node Is Nothing Then
With Node
ProgressPre.Perform()
Index += 1
Dim pid$
Dim sFolder$ = .Value("title").StringRemoveWinForbiddenSymbols
Dim storyID$ = .Value("id").Replace("highlight:", String.Empty)
If GetUserStory Then
sFolder = $"{StoriesFolder} (user)"
Else
If sFolder.IsEmptyString Then sFolder = $"Story_{storyID.IfNullOrEmpty(Index)}"
sFolder = $"{StoriesFolder}\{sFolder}"
End If
If Not storyID.IsEmptyString Then storyID &= ":"
With .Item("items")
If .ListExists Then
For Each s As EContainer In .Self
pid = storyID & s.Value("id")
If Not _TempPostsList.Contains(pid) Then
ThrowAny(Token)
ObtainMedia(s, pid, sFolder)
_TempPostsList.Add(pid)
End If
Next
End If
End With
End With
End If
End Sub
Private Function GetStoriesList() As List(Of String)
Try
UpdateRequestNumber()
Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/highlights/{ID}/highlights_tray/",, EDP.ThrowException)
If Not r.IsEmptyString Then
Dim ee As New ErrorsDescriber(EDP.ReturnValue) With {.DeclaredMessage = New MMessage($"{ToStringForLog()}:")}
@@ -1223,15 +1279,26 @@ Namespace API.Instagram
Private Sub DisableSection(ByVal Section As Object)
If Not IsNothing(Section) AndAlso TypeOf Section Is Sections Then
Dim s As Sections = DirectCast(Section, Sections)
Select Case s
Case Sections.Reels : MySiteSettings.DownloadReels.Value = False
Case Sections.Tagged : MySiteSettings.DownloadTagged.Value = False
Case Sections.Timeline, Sections.Stories, Sections.UserStories, Sections.SavedPosts
MySiteSettings.DownloadTimeline.Value = False
MySiteSettings.DownloadStories.Value = False
MySiteSettings.DownloadStoriesUser.Value = False
End Select
MyMainLOG = $"[{s}] downloading is disabled until you update your credentials".ToUpper
Dim ss As New List(Of Sections)([Enum].GetValues(GetType(Sections)).ToObjectsList(Of Sections))
If s = Sections.Reels And Not _UseGQL Then
ss.Clear()
ss.Add(s)
ElseIf s = Sections.Tagged Then
ss.Clear()
ss.Add(s)
End If
If ss.Count > 0 Then
For Each s In ss
Select Case s
Case Sections.Reels : MySiteSettings.DownloadReels.Value = False
Case Sections.Tagged : MySiteSettings.DownloadTagged.Value = False
Case Sections.Timeline, Sections.SavedPosts : MySiteSettings.DownloadTimeline.Value = False
Case Sections.Stories : MySiteSettings.DownloadStories.Value = False
Case Sections.UserStories : MySiteSettings.DownloadStoriesUser.Value = False
End Select
Next
MyMainLOG = $"[{ss.ListToStringE(, New ANumbers.EnumToStringProvider(GetType(Sections)))}] downloading is disabled until you update your credentials".ToUpper
End If
End If
End Sub
#End Region

172
SCrawler/API/JustForFans/M3U8.vb
View File

@@ -6,7 +6,7 @@
'
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports System.Threading
Imports System.Net
Imports SCrawler.API.Base
Imports PersonalUtilities.Tools
Imports PersonalUtilities.Tools.Web
@@ -18,8 +18,10 @@ Namespace API.JustForFans
Friend NotInheritable Class M3U8 : Implements IDisposable
#Region "Declarations"
Friend Const AllVid As UTypes = UTypes.m3u8 + UTypes.VideoPre
Private ReadOnly DataVideo As List(Of String)
Private ReadOnly DataAudio As List(Of String)
Private Structure M3U8URL_Indexed
Friend Index As Integer
Friend File As SFile
End Structure
Private Media As UserMedia
Private DestinationFile As SFile
Private ReadOnly Thrower As Plugin.IThrower
@@ -32,31 +34,37 @@ Namespace API.JustForFans
Private UrlAudio As String
Private FileVideo As SFile
Private FileAudio As SFile
Private FileVideo_M3U8 As SFile
Private FileAudio_M3U8 As SFile
Private ReadOnly FileVideo_IndexedParts As List(Of M3U8URL_Indexed)
Private ReadOnly FileAudio_IndexedParts As List(Of M3U8URL_Indexed)
Private RootPlaylistUrl As String
Private ReadOnly Cache As CacheKeeper
Private ReadOnly Progress As MyProgress
Private ReadOnly ProgressPre As PreProgress
Private ReadOnly ProgressExists As Boolean
Private ReadOnly UsePreProgress As Boolean
Private Property Token As CancellationToken
Private ReadOnly REGEX_FILE_EXT As RParams = RParams.DMS("[^\s""]+\.(\w+)([\?&]{1}.+|)", 1, EDP.ReturnValue)
Private ReadOnly REGEX_FILE_EXT_M4S As RParams = RParams.DM("[^\s""]+\.m4s([\?&]{1}.+|)", 0, EDP.ReturnValue)
Private ReadOnly MyFileNumberProvider As ANumbers
#End Region
#Region "Initializer"
Private Sub New(ByVal m As UserMedia, ByVal Destination As SFile, ByVal Resp As Responser, ByVal _Thrower As Plugin.IThrower,
ByVal _Progress As MyProgress, ByVal _UsePreProgress As Boolean, ByVal _Token As CancellationToken)
ByVal _Progress As MyProgress, ByVal _UsePreProgress As Boolean)
Media = m
DataVideo = New List(Of String)
DataAudio = New List(Of String)
DestinationFile = Destination
Thrower = _Thrower
'Responser = Resp
Responser = New Responser
ResponserInternal = True
FileVideo_IndexedParts = New List(Of M3U8URL_Indexed)
FileAudio_IndexedParts = New List(Of M3U8URL_Indexed)
Progress = _Progress
ProgressExists = Not Progress Is Nothing
If ProgressExists Then ProgressPre = New PreProgress(Progress)
UsePreProgress = _UsePreProgress
Token = _Token
Cache = New CacheKeeper($"{DestinationFile.PathWithSeparator}_{M3U8Base.TempCacheFolderName}\")
MyFileNumberProvider = M3U8Base.NumberProviderDefault
Cache = New CacheKeeper($"{DestinationFile.PathWithSeparator}_{M3U8Base.TempCacheFolderName}\") With {.DisposeSuspended = True}
With Cache
.CacheDeleteError = CacheDeletionError(Cache)
.DisposeSuspended = True
@@ -91,30 +99,138 @@ Namespace API.JustForFans
UrlVideo = RegexReplace(r, RParams.DMS(R_VIDEO_REGEX_PATTERN, 6, EDP.ReturnValue))
UrlAudio = RegexReplace(r, REGEX_AUDIO_URL)
If UrlVideo.IsEmptyString Then Throw New ArgumentException("Unable to identify m3u8 video track", "M3U8 video track")
Thrower.ThrowAny()
GetFiles(UrlVideo, FileVideo, False)
GetFileParts(UrlVideo, FileVideo_M3U8, FileVideo_IndexedParts, False)
Thrower.ThrowAny()
If Not UrlAudio.IsEmptyString Then GetFiles(UrlAudio, FileAudio, True)
If Not UrlAudio.IsEmptyString Then GetFileParts(UrlAudio, FileAudio_M3U8, FileAudio_IndexedParts, True)
If FileVideo_IndexedParts.Count > 0 Then _
FileVideo = GetTempFile(FileVideo_M3U8, FileVideo_IndexedParts, False, FileAudio_IndexedParts, FileAudio_IndexedParts.Count = 0)
If FileAudio_IndexedParts.Count > 0 Then _
FileAudio = GetTempFile(FileAudio_M3U8, FileAudio_IndexedParts, True, FileVideo_IndexedParts, False)
Thrower.ThrowAny()
MergeFiles()
End If
End If
End Sub
Private Sub GetFiles(ByVal URL As String, ByRef File As SFile, ByVal IsAudio As Boolean)
Private Function GetTempFile(ByVal M3U8File As SFile, ByVal IndexedList As List(Of M3U8URL_Indexed), ByVal IsAudio As Boolean,
ByVal IndexedListOther As List(Of M3U8URL_Indexed), ByVal IgnoreAudio As Boolean) As SFile
Const mapStr$ = "#EXT-X-MAP:URI"
Const extinfStr$ = "#EXTINF:"
Const m4s$ = "m4s"
Dim M3U8FileLines$() = M3U8File.GetLines
If M3U8FileLines.ListExists AndAlso IndexedList.Count > 0 AndAlso (IndexedListOther.Count > 0 Or (Not IsAudio And IgnoreAudio)) Then
Dim outputFile As SFile = $"{Cache.RootDirectory.PathWithSeparator}{IIf(IsAudio, "AUDIO.aac", "VIDEO.mp4")}"
Dim M3U8FileNew As SFile = M3U8File
M3U8FileNew.Path = IndexedList(0).File.Path
Dim v$
Dim i%, fIndx%, fIndx2%
Dim extIsm4s As Boolean
Dim LookingIndex% = -1
Dim ignoreOtherList As Boolean = IndexedListOther.Count = 0 And (Not IsAudio And IgnoreAudio)
Dim fileFinder As Predicate(Of M3U8URL_Indexed) = Function(input) input.Index = LookingIndex
Using m3u8Text As New TextSaver
For i = 0 To M3U8FileLines.Length - 1
v = M3U8FileLines(i)
If Not v.IsEmptyString Then
If v.StartsWith(mapStr) Then
LookingIndex += 1
fIndx = IndexedList.FindIndex(fileFinder)
If fIndx >= 0 Then
extIsm4s = Not IndexedList(fIndx).File.Extension.IsEmptyString AndAlso IndexedList(fIndx).File.Extension = m4s
v = v.Replace(RegexReplace(v, If(extIsm4s, REGEX_FILE_EXT_M4S, REGEX_FILE_EXT)), IndexedList(fIndx).File.File)
m3u8Text.AppendLine(v)
Else
Throw New Exception($"The map file is missing ({IIf(IsAudio, "audio", "video")})")
End If
ElseIf v.StartsWith(extinfStr) Then
LookingIndex += 1
If (i + 1) <= M3U8FileLines.Length - 1 Then
fIndx = IndexedList.FindIndex(fileFinder)
fIndx2 = If(ignoreOtherList, -1, IndexedListOther.FindIndex(fileFinder))
If fIndx >= 0 And (fIndx2 >= 0 Or ignoreOtherList) Then
If ignoreOtherList OrElse IndexedListOther(fIndx2).Index = IndexedList(fIndx).Index Then
m3u8Text.AppendLine(v)
m3u8Text.AppendLine(IndexedList(fIndx).File.File)
End If
End If
i += 1
Else
Throw New Exception($"Unexpected end of m3u8 file ({IIf(IsAudio, "audio", "video")})")
End If
Else
m3u8Text.AppendLine(v)
End If
End If
Next
m3u8Text.SaveAs(M3U8FileNew)
End Using
If M3U8FileNew.Exists Then
Using b As New BatchExecutor
AddHandler b.ErrorDataReceived, AddressOf Batch_OutputDataReceived
Thrower.ThrowAny()
ProgressChangeMax(IndexedList.Count)
b.ChangeDirectory(M3U8FileNew)
b.Execute($"""{Settings.FfmpegFile}"" -i {M3U8FileNew.File} -vcodec copy -strict -2 ""{outputFile}""")
End Using
If Not outputFile.Exists Then outputFile = Nothing
End If
Return outputFile
Else
Return Nothing
End If
End Function
Private Sub GetFileParts(ByVal URL As String, ByRef M3U8File As SFile, ByRef IndexedList As List(Of M3U8URL_Indexed), ByVal IsAudio As Boolean)
Try
Dim r$ = Responser.GetResponse(URL)
If Not r.IsEmptyString Then
Dim data As List(Of RegexMatchStruct) = RegexFields(Of RegexMatchStruct)(r, {REGEX_PLS_FILES}, {1, 2}, EDP.ReturnValue)
If data.ListExists Then
File = $"{Cache.RootDirectory.PathWithSeparator}{IIf(IsAudio, "AUDIO.aac", "VIDEO.mp4")}"
Using b As New TokenBatch(Token) With {.Encoding = Settings.CMDEncoding, .MainProcessName = "ffmpeg"}
AddHandler b.ErrorDataReceived, AddressOf Batch_OutputDataReceived
ProgressChangeMax(data.Count)
b.ChangeDirectory(Cache.RootDirectory)
b.Execute($"""{Settings.FfmpegFile}"" -i {URL} -vcodec copy -strict -2 ""{File}""")
Token.ThrowIfCancellationRequested()
If Not File.Exists Then File = Nothing
End Using
Dim appender$ = URL.Replace(URL.Split("/").LastOrDefault, String.Empty)
Dim createM3U8URL As Func(Of String, M3U8URL) =
Function(input) New M3U8URL(M3U8Base.CreateUrl(appender, input), RegexReplace(input, REGEX_FILE_EXT))
With (From d As RegexMatchStruct In data
Where Not d.Arr(0).IfNullOrEmpty(d.Arr(1)).IsEmptyString
Select createM3U8URL.Invoke(d.Arr(0).IfNullOrEmpty(d.Arr(1)).StringTrim))
If .ListExists Then
ProgressChangeMax(.Count)
M3U8File = $"{Cache.RootDirectory.PathWithSeparator}{IIf(IsAudio, "AUDIO", "VIDEO")}.m3u8"
M3U8File = TextSaver.SaveTextToFile(r, M3U8File, True)
Dim tmpCache As CacheKeeper = Cache.NewInstance
Dim dFile As SFile = tmpCache.RootDirectory
dFile.Extension = .ElementAt(0).Extension.IfNullOrEmpty("m4s")
MyFileNumberProvider.GroupSize = { .Count.ToString.Length, 3}.Max
If tmpCache.Validate Then
Using w As New WebClient
For i% = 0 To .Count - 1
Thrower.ThrowAny()
dFile.Name = $"{M3U8Base.TempFilePrefix}{i.NumToString(MyFileNumberProvider)}"
dFile.Extension = .ElementAt(i).Extension.IfNullOrEmpty(M3U8Base.TempFileDefaultExtension)
Try
ProgressPerform()
w.DownloadFile(.ElementAt(i).URL, dFile)
tmpCache.AddFile(dFile, True)
IndexedList.Add(New M3U8URL_Indexed With {.File = dFile, .Index = i})
Catch down_oex As OperationCanceledException
Throw down_oex
Catch down_dex As ObjectDisposedException
Throw down_dex
Catch ex As Exception
End Try
Next
End Using
Else
Throw New Exception("Can't create cache directory")
End If
End If
End With
End If
End If
Catch oex As OperationCanceledException
@@ -123,7 +239,7 @@ Namespace API.JustForFans
Throw dex
Catch ex As Exception
ErrorsDescriber.Execute(EDP.SendToLog + EDP.ThrowException, ex,
$"API.JustForFans.M3U8.GetFiles({IIf(IsAudio, "audio", "video")}):{vbCr}URL: {URL}{vbCr}File: {File}")
$"API.JustForFans.M3U8.GetFileParts({IIf(IsAudio, "audio", "video")}):{vbCr}URL: {URL}{vbCr}Post: {Media.URL_BASE}")
End Try
End Sub
Private Async Sub Batch_OutputDataReceived(ByVal Sender As Object, ByVal e As DataReceivedEventArgs)
@@ -135,8 +251,10 @@ Namespace API.JustForFans
Dim f As SFile = SFile.IndexReindex(DestinationFile,,, p, EDP.ReturnValue).IfNullOrEmpty(DestinationFile)
If Not FileVideo.IsEmptyString And Not FileAudio.IsEmptyString Then
DestinationFile = FFMPEG.MergeFiles({FileVideo, FileAudio}, Settings.FfmpegFile, f, Settings.CMDEncoding, p, EDP.ThrowException)
Else
ElseIf FileVideo.Exists Then
If Not SFile.Move(FileVideo, f) Then DestinationFile = FileVideo
Else
Throw New Exception($"Unable to download file ({Media.URL_BASE})")
End If
Catch ex As Exception
ErrorsDescriber.Execute(EDP.SendToLog + EDP.ThrowException, ex, $"[M3U8.MergeFiles]")
@@ -165,8 +283,8 @@ Namespace API.JustForFans
#End Region
#Region "Static Download"
Friend Shared Function Download(ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Resp As Responser, ByVal Thrower As Plugin.IThrower,
ByVal Progress As MyProgress, ByVal UsePreProgress As Boolean, ByVal _Token As CancellationToken) As SFile
Using m As New M3U8(Media, DestinationFile, Resp, Thrower, Progress, UsePreProgress, _Token)
ByVal Progress As MyProgress, ByVal UsePreProgress As Boolean) As SFile
Using m As New M3U8(Media, DestinationFile, Resp, Thrower, Progress, UsePreProgress)
m.Download()
If m.DestinationFile.Exists Then Return m.DestinationFile Else Return Nothing
End Using
@@ -177,8 +295,8 @@ Namespace API.JustForFans
Private Overloads Sub Dispose(ByVal disposing As Boolean)
If Not disposedValue Then
If disposing Then
DataVideo.Clear()
DataAudio.Clear()
FileVideo_IndexedParts.Clear()
FileAudio_IndexedParts.Clear()
ProgressPre.DisposeIfReady
Cache.Dispose()
If ResponserInternal Then Responser.DisposeIfReady

2
SCrawler/API/JustForFans/UserData.vb
View File

@@ -336,7 +336,7 @@ Namespace API.JustForFans
DownloadContentDefault(Token)
End Sub
Protected Overrides Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Token As CancellationToken) As SFile
Return M3U8.Download(Media, DestinationFile, ResponserNoHandlers, Me, Progress, Not IsSingleObjectDownload, Token)
Return M3U8.Download(Media, DestinationFile, ResponserNoHandlers, Me, Progress, Not IsSingleObjectDownload)
End Function
#End Region
#Region "DownloadSingleObject"

31
SCrawler/API/ThreadsNet/SiteSettings.vb
View File

@@ -68,6 +68,19 @@ Namespace API.ThreadsNet
End If
End Sub
#End Region
#Region "Other properties"
<PropertyOption(ControlText:="Request timer (any)",
ControlToolTip:="The timer (in milliseconds) that SCrawler should wait before executing the next request." &
vbCr & "The default value is 1'000." & vbCr & "The minimum value is 0." & IG.TimersUrgentTip, AllowNull:=False),
PXML, PClonable>
Friend ReadOnly Property RequestsWaitTimer_Any As PropertyValue
<Provider(NameOf(RequestsWaitTimer_Any), FieldsChecker:=True)>
Private ReadOnly Property RequestsWaitTimer_AnyProvider As IFormatProvider
<PropertyOption(ControlText:="Download data",
ControlToolTip:="The internal value indicates that site data should be downloaded." & vbCr &
"It becomes unchecked when the site returns an error."), PXML>
Friend ReadOnly Property DownloadData_Impl As PropertyValue
#End Region
#End Region
#Region "Initializer"
Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
@@ -112,7 +125,7 @@ Namespace API.ThreadsNet
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "cors"))
.Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "same-origin"))
.Add("Sec-Fetch-User", "?1")
.Add(DeclaredNames.Header_FB_FRIENDLY_NAME, "BarcelonaProfileThreadsTabRefetchableQuery")
.Add(Instagram.UserData.GQL_HEADER_FB_FRINDLY_NAME, "BarcelonaProfileThreadsTabRefetchableQuery")
End With
.CookiesExtractMode = Responser.CookiesExtractModes.Any
.CookiesUpdateMode = CookieKeeper.UpdateModes.ReplaceByNameAll
@@ -129,6 +142,10 @@ Namespace API.ThreadsNet
HH_PLATFORM_VER = New PropertyValue(platform, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_PLATFORM_VER), v))
HH_USER_AGENT = New PropertyValue(useragent, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_USER_AGENT), v))
RequestsWaitTimer_Any = New PropertyValue(1000)
RequestsWaitTimer_AnyProvider = New IG.TimersChecker(0)
DownloadData_Impl = New PropertyValue(True)
UrlPatternUser = "https://www.threads.net/@{0}"
UserRegex = RParams.DMS(String.Format(UserRegexDefaultPattern, "threads.net/@"), 1)
ImageVideoContains = "threads.net"
@@ -155,7 +172,7 @@ Namespace API.ThreadsNet
#End Region
#Region "BaseAuthExists, GetUserUrl, GetUserPostUrl"
Friend Overrides Function BaseAuthExists() As Boolean
Return Responser.CookiesExists And {HH_CSRF_TOKEN, HH_IG_APP_ID}.All(Function(v) ACheck(Of String)(v.Value))
Return Responser.CookiesExists And {HH_CSRF_TOKEN, HH_IG_APP_ID}.All(Function(v) ACheck(Of String)(v.Value)) And CBool(DownloadData_Impl.Value)
End Function
Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String
Return String.Format(UrlPatternUser, DirectCast(User, UserData).NameTrue)
@@ -171,13 +188,23 @@ Namespace API.ThreadsNet
End Function
#End Region
#Region "Update"
Private __Cookies As CookieKeeper = Nothing
Friend Overrides Sub BeginEdit()
__Cookies = Responser.Cookies.Copy
MyBase.BeginEdit()
End Sub
Friend Overrides Sub Update()
If _SiteEditorFormOpened And Responser.CookiesExists Then
Dim csrf$ = If(Responser.Cookies.FirstOrDefault(Function(c) c.Name.StringToLower = IG.Header_CSRF_TOKEN_COOKIE)?.Value, String.Empty)
If Not csrf.IsEmptyString Then HH_CSRF_TOKEN.Value = csrf
If Not __Cookies Is Nothing AndAlso Not __Cookies.ListEquals(Responser.Cookies) Then DownloadData_Impl.Value = True
End If
MyBase.Update()
End Sub
Friend Overrides Sub EndEdit()
__Cookies.DisposeIfReady
MyBase.EndEdit()
End Sub
#End Region
End Class
End Namespace

71
SCrawler/API/ThreadsNet/UserData.vb
View File

@@ -24,11 +24,9 @@ Namespace API.ThreadsNet
End Get
End Property
Private ReadOnly DefaultParser_ElemNode_Default() As Object = {"node", "thread_items", 0, "post"}
Private OPT_LSD As String = String.Empty
Private OPT_FB_DTSG As String = String.Empty
Private ReadOnly Property Valid As Boolean
Get
Return Not OPT_LSD.IsEmptyString And Not OPT_FB_DTSG.IsEmptyString And Not ID.IsEmptyString
Return ValidateBaseTokens() And Not ID.IsEmptyString
End Get
End Property
#End Region
@@ -54,23 +52,31 @@ Namespace API.ThreadsNet
End Sub
#End Region
#Region "Download functions"
Private Sub WaitTimer()
If CInt(MySettings.RequestsWaitTimer_Any.Value) > 0 Then Thread.Sleep(CInt(MySettings.RequestsWaitTimer_Any.Value))
End Sub
Private Sub DisableDownload()
MySettings.DownloadData_Impl.Value = False
MyMainLOG = $"{Site} downloading is disabled until you update your credentials"
End Sub
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
Dim errorFound As Boolean = False
Try
Responser.Method = "POST"
LoadSavePostsKV(True)
OPT_LSD = String.Empty
OPT_FB_DTSG = String.Empty
DownloadData(String.Empty, Token)
Catch ex As Exception
errorFound = True
Throw ex
Finally
Responser.Method = "POST"
UpdateResponser()
MySettings.UpdateResponserData(Responser)
If Not errorFound Then LoadSavePostsKV(False)
End Try
If CBool(MySettings.DownloadData_Impl.Value) Then
Dim errorFound As Boolean = False
Try
Responser.Method = "POST"
LoadSavePostsKV(True)
ResetBaseTokens()
DownloadData(String.Empty, Token)
Catch ex As Exception
errorFound = True
Throw ex
Finally
Responser.Method = "POST"
UpdateResponser()
MySettings.UpdateResponserData(Responser)
If Not errorFound Then LoadSavePostsKV(False)
End Try
End If
End Sub
Protected Overrides Sub UpdateResponser()
If Not Responser Is Nothing AndAlso Not Responser.Disposed Then
@@ -95,11 +101,11 @@ Namespace API.ThreadsNet
UpdateCredentials()
If idIsNull And Not ID.IsEmptyString Then _ForceSaveUserInfo = True
End If
If Not Valid Then Throw New Plugin.ExitException("Some credentials are missing")
If Not Valid Then DisableDownload() : Throw New Plugin.ExitException("Some credentials are missing")
Responser.Method = "POST"
Responser.Referer = $"https://www.threads.net/@{NameTrue}"
Responser.Headers.Add(Header_FB_LSD, OPT_LSD)
Responser.Headers.Add(GQL_HEADER_FB_LSD, Token_lsd)
Dim nextCursor$ = String.Empty
Dim dataFound As Boolean = False
@@ -112,7 +118,7 @@ Namespace API.ThreadsNet
End If
vars = SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & vars & "}")
URL = String.Format(urlPattern, OPT_LSD, vars, SymbolsConverter.ASCII.EncodeSymbolsOnly(OPT_FB_DTSG))
URL = String.Format(urlPattern, Token_lsd, vars, Token_dtsg_Var)
Using j As EContainer = GetDocument(URL, Token)
If j.ListExists Then
@@ -135,8 +141,9 @@ Namespace API.ThreadsNet
Private Function GetDocument(ByVal URL As String, ByVal Token As CancellationToken, Optional ByVal Round As Integer = 0) As EContainer
Try
ThrowAny(Token)
If Round > 0 AndAlso Not UpdateCredentials() Then Throw New Exception("Failed to update credentials")
If Round > 0 AndAlso Not UpdateCredentials() Then DisableDownload() : Throw New Exception("Failed to update credentials")
ThrowAny(Token)
WaitTimer()
Dim r$ = Responser.GetResponse(URL)
If Not r.IsEmptyString Then Return JsonDocument.Parse(r) Else Throw New Exception("Failed to get a response")
Catch ex As Exception
@@ -149,12 +156,12 @@ Namespace API.ThreadsNet
End Function
Private Function UpdateCredentials(Optional ByVal e As ErrorsDescriber = Nothing) As Boolean
Dim URL$ = $"https://www.threads.net/@{NameTrue}"
OPT_LSD = String.Empty
OPT_FB_DTSG = String.Empty
ResetBaseTokens()
Try
Responser.Method = "GET"
Responser.Referer = URL
Responser.Headers.Remove(Header_FB_LSD)
Responser.Headers.Remove(GQL_HEADER_FB_LSD)
WaitTimer()
Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException)
Dim rr As RParams
Dim tt$, ttVal$
@@ -168,15 +175,15 @@ Namespace API.ThreadsNet
.WhatGet = RegexReturn.Value
End With
For Each tt In tokens
If Not OPT_FB_DTSG.IsEmptyString And Not OPT_LSD.IsEmptyString Then
If Not Token_dtsg.IsEmptyString And Not Token_lsd.IsEmptyString Then
Exit For
Else
ttVal = RegexReplace(tt, rr)
If Not ttVal.IsEmptyString Then
If ttVal.Contains(":") Then
If OPT_FB_DTSG.IsEmptyString Then OPT_FB_DTSG = ttVal
If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal
Else
If OPT_LSD.IsEmptyString Then OPT_LSD = ttVal
If Token_lsd.IsEmptyString Then Token_lsd = ttVal
End If
End If
End If
@@ -187,9 +194,9 @@ Namespace API.ThreadsNet
Return Valid
Catch ex As Exception
Dim notFound$ = String.Empty
If OPT_FB_DTSG.IsEmptyString Then notFound.StringAppend(Header_FB_LSD)
If OPT_LSD.IsEmptyString Then notFound.StringAppend("lsd")
ValidateBaseTokens(notFound)
If ID.IsEmptyString Then notFound.StringAppend("User ID")
DisableDownload()
Dim eex As New ErrorsDescriberException($"{ToStringForLog()}: failed to update some{IIf(notFound.IsEmptyString, String.Empty, $" ({notFound})")} credentials",,, ex) With {
.ReplaceMainMessage = True,
.SendToLogOnlyMessage = Responser.StatusCode = Net.HttpStatusCode.InternalServerError And Responser.Status = Net.WebExceptionStatus.ProtocolError
@@ -224,7 +231,7 @@ Namespace API.ThreadsNet
If m.State = UserMedia.States.Missing And Not m.Post.ID.IsEmptyString Then
ThrowAny(Token)
vars = SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & String.Format(varsPattern, m.Post.ID.Split("_").FirstOrDefault, ID) & "}")
URL = String.Format(urlPattern, OPT_LSD, vars, SymbolsConverter.ASCII.EncodeSymbolsOnly(OPT_FB_DTSG))
URL = String.Format(urlPattern, Token_lsd, vars, Token_dtsg_Var)
j = GetDocument(URL, Token)
If j.ListExists Then

11
SCrawler/Download/Groups/DownloadGroupCollection.vb
View File

@@ -37,7 +37,7 @@ Namespace DownloadObjects.Groups
End If
End With
End If
GroupsList.ListReindex
Reindex()
End Sub
#End Region
#Region "Base properties"
@@ -75,7 +75,10 @@ Namespace DownloadObjects.Groups
GroupsList.Sort()
End Sub
Friend Sub Reindex()
Dim initUpValue As Boolean = _UpdateMode
BeginUpdate()
GroupsList.ListReindex
If Not initUpValue Then EndUpdate()
End Sub
#End Region
#Region "Group handlers"
@@ -92,7 +95,7 @@ Namespace DownloadObjects.Groups
If i >= 0 Then
GroupsList(i).Dispose()
GroupsList.RemoveAt(i)
GroupsList.ListReindex
Reindex()
Update()
End If
End Sub
@@ -133,9 +136,9 @@ Namespace DownloadObjects.Groups
AddHandler .Deleted, AddressOf OnGroupDeleted
AddHandler .Updated, AddressOf OnGroupUpdated
If Not exists Then
GroupsList.ListReindex
Reindex()
GroupsList.Sort()
GroupsList.ListReindex
Reindex()
If Not Item.IsViewFilter And Not _UpdateMode Then RaiseEvent Added(.Self)
Else
If Not Item.IsViewFilter And Not _UpdateMode Then RaiseEvent Updated(.Self)

24
SCrawler/Download/TDownloader.vb
View File

@@ -192,10 +192,6 @@ Namespace DownloadObjects
_FilesSessionCleared = True
Dim files As List(Of SFile) = SFile.GetFiles(SessionsPath.CSFileP, "*.xml",, EDP.ReturnValue)
If files.ListExists Then files.RemoveAll(Settings.Feeds.FeedSpecialRemover)
If RenameOldFileNames(files) Then
files = SFile.GetFiles(SessionsPath.CSFileP, "*.xml",, EDP.ReturnValue)
If files.ListExists Then files.RemoveAll(Settings.Feeds.FeedSpecialRemover)
End If
If files.ListExists Then
Const ds$ = "yyyyMMdd"
Dim nd$ = Now.ToString(ds), d1$ = Now.AddDays(-1).ToString(ds), d2$ = Now.AddDays(-2).ToString(ds)
@@ -211,26 +207,6 @@ Namespace DownloadObjects
ErrorsDescriber.Execute(EDP.SendToLog, ex, "[DownloadObjects.TDownloader.ClearSessions]")
End Try
End Sub
Private Function RenameOldFileNames(ByVal files As List(Of SFile)) As Boolean
Dim result As Boolean = False
Try
If files.ListExists AndAlso files.Exists(Function(ff) ff.Name.StringToLower.StartsWith("latest")) Then
Dim d As Date
Dim fileCurrent As SFile, fileNew As SFile
For Each fileCurrent In files
If fileCurrent.Name.StringToLower.StartsWith("latest") Then
d = IO.File.GetLastWriteTime(fileCurrent)
fileNew = fileCurrent
fileNew.Name = AConvert(Of String)(d, SessionDateTimeProvider)
SFile.Rename(fileCurrent, fileNew,, EDP.None)
result = True
End If
Next
End If
Catch
End Try
Return result
End Function
#End Region
Friend ReadOnly Property Downloaded As List(Of IUserData)
Private ReadOnly NProv As IFormatProvider

15
SCrawler/Editors/SiteEditorForm.vb
View File

@@ -138,6 +138,7 @@ Namespace Editors
End Function
End Class
#End Region
Private ReadOnly PropertyValid As Predicate(Of PropertyValueHost) = Function(p) (Not p.IsHidden Or SiteSettingsShowHiddenControls) And Not p.Options Is Nothing
Private ReadOnly Property Host As SettingsHost
Private Property HostCollection As SettingsHostCollection
Friend Sub New(ByVal h As SettingsHost)
@@ -147,7 +148,7 @@ Namespace Editors
If Not Host.Responser Is Nothing Then Cookies = Host.Responser.Cookies.Copy
LBL_AUTH = New Label With {.Text = "Authorization", .TextAlign = ContentAlignment.MiddleCenter, .Dock = DockStyle.Fill}
LBL_OTHER = New Label With {.Text = "Other Parameters", .TextAlign = ContentAlignment.MiddleCenter, .Dock = DockStyle.Fill}
Host.Source.BeginEdit()
Host.BeginEdit()
End Sub
Private Sub SiteEditorForm_Load(sender As Object, e As EventArgs) Handles Me.Load
Try
@@ -216,7 +217,7 @@ Namespace Editors
If .PropList.Exists(Function(p) p.ControlNumber >= 0) Then .PropList.Sort()
For Each pAuth As Boolean In pArr
For Each prop As PropertyValueHost In .PropList
If Not prop.Options Is Nothing Then
If PropertyValid.Invoke(prop) Then
With prop
If .Options.IsAuth = pAuth Then
@@ -286,7 +287,7 @@ Namespace Editors
If Not SpecialButton Is Nothing Then SpecialButton.Dispose()
LBL_AUTH.Dispose()
LBL_OTHER.Dispose()
Host.Source.EndEdit()
Host.EndEdit()
If Not Cookies Is Nothing Then Cookies.Dispose()
End Sub
Private Sub MyDefs_ButtonOkClick(ByVal Sender As Object, ByVal e As KeyHandleEventArgs) Handles MyDefs.ButtonOkClick
@@ -311,8 +312,6 @@ Namespace Editors
Next
End If
Settings.BeginUpdate()
SiteDefaultsFunctions.SetPropByChecker(TP_SITE_PROPS, Host)
If TXT_PATH.IsEmptyString Then .Path = Nothing Else .Path = TXT_PATH.Text
.SavedPostsPath = TXT_PATH_SAVED_POSTS.Text
@@ -327,13 +326,11 @@ Namespace Editors
End With
End If
If .PropList.Count > 0 Then .PropList.ForEach(Sub(p) If Not p.Options Is Nothing Then p.UpdateValueByControl())
If .PropList.Count > 0 Then .PropList.ForEach(Sub(p) If PropertyValid.Invoke(p) Then p.UpdateValueByControl())
.Source.Update()
.Update()
End With
Settings.EndUpdate()
MyDefs.CloseForm()
End If
End Sub

4
SCrawler/MainFrame.Designer.vb generated
View File

@@ -420,9 +420,7 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
Me.BTT_DOWN_SPEC.Name = "BTT_DOWN_SPEC"
Me.BTT_DOWN_SPEC.Size = New System.Drawing.Size(221, 22)
Me.BTT_DOWN_SPEC.Text = "Download (advanced)"
Me.BTT_DOWN_SPEC.ToolTipText = "Filter the users you want to download and download them." & Global.Microsoft.VisualBasic.ChrW(13) & Global.Microsoft.VisualBasic.ChrW(10) & "Shift+Click to download" &
", including non-existent users." & Global.Microsoft.VisualBasic.ChrW(13) & Global.Microsoft.VisualBasic.ChrW(10) & "Ctrl+Shift+Click to download, excluding from th" &
"e feed, including non-existent users."
Me.BTT_DOWN_SPEC.ToolTipText = "Filter the users you want to download and download them."
'
'BTT_DOWN_VIDEO
'

1
SCrawler/MainMod.vb
View File

@@ -81,6 +81,7 @@ Friend Module MainMod
Friend ReadOnly FeedVideoLengthProvider As New ADateTime("hh\:mm\:ss") With {.TimeParseMode = ADateTime.TimeModes.TimeSpan}
Friend ReadOnly LogConnector As New LogHost
Friend DefaultUserAgent As String = String.Empty
Friend SiteSettingsShowHiddenControls As Boolean = False
#Region "NonExistingUsersLog"
Friend ReadOnly NonExistingUsersLog As New TextSaver($"LOGs\NonExistingUsers.txt") With {.LogMode = True, .AutoSave = True}
Friend Sub AddNonExistingUserToLog(ByVal Message As String)

8
SCrawler/PluginsEnvironment/Attributes/Attributes.vb
View File

@@ -45,4 +45,12 @@ Namespace Plugin.Attributes
Public Clone As Boolean = True
Public Update As Boolean = True
End Class
Public Class HiddenControlAttribute : Inherits Attribute
Public ReadOnly IsHidden As Boolean = True
Public Sub New()
End Sub
Public Sub New(ByVal _IsHidden As Boolean)
IsHidden = _IsHidden
End Sub
End Class
End Namespace

2
SCrawler/PluginsEnvironment/Hosts/DownloadableMediaHost.vb
View File

@@ -125,7 +125,7 @@ Namespace Plugin.Hosts
Dim __url$ = DirectCast(Me, IDownloadableMedia).URL_BASE.IfNullOrEmpty(URL)
If File.Exists And Not __url.IsEmptyString And MyDownloaderSettings.CreateUrlFiles Then
Dim urlFile As SFile = CreateUrlFile(__url, File)
If urlFile.Exists Then Files.Add(urlFile)
If urlFile.Exists Then AddFile(urlFile)
End If
If Not ExternalSource Is Nothing Then
With ExternalSource : _HasError = .HasError : _Exists = .Exists : End With

2
SCrawler/PluginsEnvironment/Hosts/PropertyValueHost.vb
View File

@@ -46,6 +46,7 @@ Namespace Plugin.Hosts
End Property
Friend ReadOnly IsTaskCounter As Boolean
Friend ReadOnly Exists As Boolean = False
Friend ReadOnly IsHidden As Boolean = False
#Region "XML"
Private ReadOnly _XmlName As String
Private ReadOnly _XmlNameChecked As String
@@ -309,6 +310,7 @@ Namespace Plugin.Hosts
UpdateMember()
Options = Member.GetCustomAttribute(Of PropertyOption)()
IsTaskCounter = Not Member.GetCustomAttribute(Of TaskCounter)() Is Nothing
IsHidden = If(Member.GetCustomAttribute(Of HiddenControlAttribute)?.IsHidden, False)
With Member.GetCustomAttribute(Of PXML)
If Not .Self Is Nothing Then
_XmlName = .ElementName

53
SCrawler/PluginsEnvironment/Hosts/SettingsHost.vb
View File

@@ -23,6 +23,9 @@ Namespace Plugin.Hosts
Friend Event Deleted As SettingsHostActionEventHandler
Friend Event OkClick As SettingsHostActionEventHandler
Friend Event CloneClick As SettingsHostActionEventHandler
Friend Event OnBeginEdit As SettingsHostActionEventHandler
Friend Event OnEndEdit As SettingsHostActionEventHandler
Friend Event OnUpdate As SettingsHostActionEventHandler
#End Region
#Region "Controls"
Private WithEvents BTT_SETTINGS As ToolStripMenuItem
@@ -258,7 +261,11 @@ Namespace Plugin.Hosts
Source = Plugin
Source.Logger = LogConnector
[Default] = IsDef
If _XML Is Nothing Then IsAbstract = True
If _XML Is Nothing Then
IsAbstract = True
Else
_XML.BeginUpdate()
End If
PropList = New List(Of PropertyValueHost)
@@ -289,11 +296,11 @@ Namespace Plugin.Hosts
End If
Next
End If
If _Key = API.PathPlugin.PluginKey And Not _XML Is Nothing Then _XML.XmlReadOnly = True
Dim i%
Dim n() As String = {SettingsCLS.Name_Node_Sites, Name}
_AccountName = New XMLValue(Of String)(NameXML_AccountName,, _XML, n)
_AccountName = New XMLValue(Of String)(NameXML_AccountName,, _XML)
Source.AccountName = _AccountName
Source.BeginInit()
@@ -373,34 +380,42 @@ Namespace Plugin.Hosts
PropList.ForEach(Sub(p) p.SetDependents(PropList))
End If
_Path = New XMLValue(Of SFile)("Path",, _XML, n, New XMLToFilePathProvider)
_SavedPostsPath = New XMLValue(Of SFile)("SavedPostsPath",, _XML, n, New XMLToFilePathProvider)
DownloadSavedPosts = New XMLValue(Of Boolean)("DownloadSavedPosts", True, _XML, n)
_Path = New XMLValue(Of SFile)("Path",, _XML,, New XMLToFilePathProvider)
_SavedPostsPath = New XMLValue(Of SFile)("SavedPostsPath",, _XML,, New XMLToFilePathProvider)
DownloadSavedPosts = New XMLValue(Of Boolean)("DownloadSavedPosts", True, _XML)
Temporary = New XMLValue(Of Boolean)
Temporary.SetExtended("Temporary", False, _XML, n)
Temporary.SetExtended("Temporary", False, _XML)
Temporary.SetDefault(_Temp)
Temporary.Update()
DownloadImages = New XMLValue(Of Boolean)
DownloadImages.SetExtended("DownloadImages", True, _XML, n)
DownloadImages.SetExtended("DownloadImages", True, _XML)
DownloadImages.SetDefault(_Imgs)
DownloadImages.Update()
DownloadVideos = New XMLValue(Of Boolean)
DownloadVideos.SetExtended("DownloadVideos", True, _XML, n)
DownloadVideos.SetExtended("DownloadVideos", True, _XML)
DownloadVideos.SetDefault(_Vids)
DownloadVideos.Update()
DownloadSiteData = New XMLValue(Of Boolean)("DownloadSiteData", True, _XML, n)
DownloadSiteData = New XMLValue(Of Boolean)("DownloadSiteData", True, _XML)
GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n)
GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML)
If PropList.Count > 0 Then
Dim MaxOffset% = Math.Max(PropList.Max(Function(pp) pp.LeftOffset), PropertyValueHost.LeftOffsetDefault)
For Each p As PropertyValueHost In PropList
If Not IsAbstract Then p.SetXmlEnvironment(_XML, n)
If Not IsAbstract Then p.SetXmlEnvironment(_XML)
p.LeftOffset = MaxOffset
Next
End If
Source.EndInit()
If Not _XML Is Nothing Then
_XML.EndUpdate()
If _XML.ChangesDetected Then _XML.UpdateData(EDP.SendToLog)
End If
End Sub
Friend Function Apply(ByVal _XML As XmlFile, ByVal GlobalPath As SFile,
ByRef _Temp As XMLValue(Of Boolean), ByRef _Imgs As XMLValue(Of Boolean), ByRef _Vids As XMLValue(Of Boolean)) As SettingsHost
@@ -522,6 +537,20 @@ Namespace Plugin.Hosts
Private Function ConvertUser(ByVal User As IUserData) As Object
Return If(DirectCast(User, UserDataBase).ExternalPlugin, User)
End Function
#Region "Edit"
Friend Sub BeginEdit()
Source.BeginEdit()
RaiseEvent OnBeginEdit(Me)
End Sub
Friend Sub Update()
Source.Update()
RaiseEvent OnUpdate(Me)
End Sub
Friend Sub EndEdit()
Source.EndEdit()
RaiseEvent OnEndEdit(Me)
End Sub
#End Region
#End Region
#Region "IEquatable Support"
Friend Overloads Function Equals(ByVal Other As SettingsHost) As Boolean Implements IEquatable(Of SettingsHost).Equals

62
SCrawler/PluginsEnvironment/Hosts/SettingsHostCollection.vb
View File

@@ -91,19 +91,49 @@ Namespace Plugin.Hosts
End If
End With
HostsUnavailableIndexes = New List(Of Integer)
Hosts = New List(Of SettingsHost) From {New SettingsHost(CreateInstance(), True, _XML, GlobalPath, _Temp, _Imgs, _Vids)}
HostsXml = New List(Of XmlFile)
Dim defInstance As ISiteSettings = CreateInstance()
HostsXml = New List(Of XmlFile) From {
GetNewXmlFile($"{SettingsFolderName}\{SiteSettingsBase.ResponserFilePrefix}{defInstance.Site}_Settings.xml", defInstance.Site, _XML)
}
Hosts = New List(Of SettingsHost) From {New SettingsHost(defInstance, True, HostsXml(0), GlobalPath, _Temp, _Imgs, _Vids)}
Dim hostFiles As List(Of SFile) = SFile.GetFiles(SettingsFolderName.CSFileP, $"{String.Format(FileNamePattern, Key, Name)}*.xml",, EDP.ReturnValue)
If hostFiles.ListExists Then
For Each f As SFile In hostFiles
HostsXml.Add(New XmlFile(f) With {.AutoUpdateFile = True})
Hosts.Add(New SettingsHost(CreateInstance(HostsXml.Last.Value({SettingsCLS.Name_Node_Sites, [Default].Name}, SettingsHost.NameXML_AccountName)), False, HostsXml.Last,
HostsXml.Add(GetNewXmlFile(f, [Default].Name))
Hosts.Add(New SettingsHost(CreateInstance(HostsXml.Last.Value(SettingsHost.NameXML_AccountName)), False, HostsXml.Last,
GlobalPath, _Temp, _Imgs, _Vids))
Next
End If
Hosts.ListReindex
Hosts.ForEach(Sub(h) SetHostHandlers(h))
End Sub
Private Function GetNewXmlFile(ByVal f As SFile, ByVal SiteName As String, Optional ByVal SourceXml As XmlFile = Nothing) As XmlFile
Dim x As New XmlFile(f,, False) With {.AutoUpdateFile = True}
If Not f.Exists Then x.Name = "SiteSettings"
x.LoadData()
'URGENT: reorganization of settings: remove the following code
Dim n$() = {SettingsCLS.Name_Node_Sites, SiteName}
Dim processed As Boolean = False
With If(SourceXml, x)
If .Count > 0 AndAlso .Contains(n) Then
With .Item(n)
If .ListExists Then
For Each container As EContainer In .Self : x.Add(container.Name, container.Value) : Next
processed = True
End If
End With
If processed Then
.Remove(n)
If SourceXml Is Nothing Then .Remove(SettingsCLS.Name_Node_Sites)
x.Name = "SiteSettings"
x.UpdateData()
End If
End If
End With
'-----END REMOVE-----
Return x
End Function
#End Region
#Region "CreateInstance"
Private Function CreateInstance(Optional ByVal Name As String = Nothing, Optional ByVal Abstract As Boolean = False) As ISiteSettings
@@ -121,15 +151,33 @@ Namespace Plugin.Hosts
End Function
#End Region
#Region "Host handlers"
#Region "Edit"
Private Sub Hosts_OnBeginEdit(ByVal Obj As SettingsHost)
If Obj.Index.ValueBetween(0, HostsXml.Count - 1) Then HostsXml(Obj.Index).BeginUpdate()
End Sub
Private Sub Hosts_OnUpdate(ByVal Obj As SettingsHost)
End Sub
Private Sub Hosts_OnEndEdit(ByVal Obj As SettingsHost)
If Obj.Index.ValueBetween(0, HostsXml.Count - 1) Then
With HostsXml(Obj.Index)
.EndUpdate()
If .ChangesDetected Then .UpdateData(EDP.SendToLog)
End With
End If
End Sub
#End Region
Private Sub SetHostHandlers(ByVal Host As SettingsHost)
AddHandler Host.OkClick, AddressOf Hosts_OkClick
AddHandler Host.Deleted, AddressOf Hosts_Deleted
AddHandler Host.CloneClick, AddressOf Hosts_CloneClick
AddHandler Host.OnBeginEdit, AddressOf Hosts_OnBeginEdit
AddHandler Host.OnUpdate, AddressOf Hosts_OnUpdate
AddHandler Host.OnEndEdit, AddressOf Hosts_OnEndEdit
If Host.Index > 0 Then Host.Source.DefaultInstance = [Default].Source : Host.DefaultInstanceChanged()
End Sub
Private Sub Hosts_OkClick(ByVal Obj As SettingsHost)
If Obj.Index = -1 Then
HostsXml.Add(New XmlFile($"{SettingsFolderName}\{String.Format(FileNamePatternFull, Key, Name, Obj.AccountName)}.xml") With {.AutoUpdateFile = True})
HostsXml.Add(GetNewXmlFile($"{SettingsFolderName}\{String.Format(FileNamePatternFull, Key, Name, Obj.AccountName)}.xml", Name))
With Settings : Hosts.Add(Obj.Apply(HostsXml.Last, .GlobalPath,
.DefaultTemporary, .DefaultDownloadImages, .DefaultDownloadVideos)) : End With
HostsXml.Last.UpdateData()
@@ -157,11 +205,11 @@ Namespace Plugin.Hosts
MsgBoxE({$"An error occurred while changing user accounts (see log for details).{vbCr}Operation canceled.", ChngUACC_MsgTitle}, vbCritical)
Exit Sub
End Select
With HostsXml(Obj.Index - 1)
With HostsXml(Obj.Index)
.File.Delete(SFO.File, SFODelete.DeleteToRecycleBin, EDP.None)
.Dispose()
End With
HostsXml.RemoveAt(Obj.Index - 1)
HostsXml.RemoveAt(Obj.Index)
Hosts.RemoveAt(Obj.Index)
Hosts.ListReindex
Obj.Source.Delete()

1
SCrawler/SCrawler.vbproj
View File

@@ -181,6 +181,7 @@
<Compile Include="API\Facebook\UserData.vb" />
<Compile Include="API\Facebook\UserExchangeOptions.vb" />
<Compile Include="API\Instagram\EditorExchangeOptions.vb" />
<Compile Include="API\Instagram\UserData.GQL.vb" />
<Compile Include="API\JustForFans\Declarations.vb" />
<Compile Include="API\JustForFans\M3U8.vb" />
<Compile Include="API\JustForFans\SiteSettings.vb" />

4
SCrawler/SettingsCLS.vb
View File

@@ -178,6 +178,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
Friend Property FeedViews As FeedViewCollection
Private ReadOnly BlackListFile As SFile = $"{SettingsFolderName}\BlackList.txt"
Private ReadOnly UsersSettingsFile As SFile = $"{SettingsFolderName}\Users.xml"
Private ReadOnly Property SettingsVersion As XMLValue(Of Integer)
#End Region
#Region "Initializer"
Friend Sub New()
@@ -201,6 +202,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
EnvironmentProgramsList = New List(Of String)
AutomationFile = New XMLValue(Of String)("AutomationFile",, MyXML)
SiteSettingsShowHiddenControls = MyXML.Value("SiteSettingsShowHiddenControls").FromXML(Of Boolean)(False)
Dim n() As String
Dim n_old() As String 'URGENT: remove this line
@@ -210,6 +212,8 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
Dim forceSaveXML As Boolean = Not SettingsReoranized 'URGENT: remove this line
Dim forceSaveXML2 As Boolean = Not SettingsReoranized OrElse Not SettingsReoranized2 'URGENT: remove this line
SettingsVersion = New XMLValue(Of Integer)("SettingsVersion", 0, MyXML)
#Region "Properties: environment"
'Environment
n = {"MediaEnvironment"}