mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-15 16:22:17 +00:00
2023.3.1.0
Add 'Path' plugin UserDataBase: changed file names for saved posts; removed 'Self' property; add 'MyFileSettings' field; added UserSiteName; changed download envir algo Twitter: added MD5 comparison; duplicate images removal option; UserSiteName parsing; download icon and banner Instagram: added a new option for token 'www_claim'; removed requirement of token 'www_claim'; UserSiteName parsing; download icon Reddit: UserSiteName parsing; download icon and banner PornHub: fixed unicode titles XHamster: added channels ffmpeg: fixed max input length error during files combining; fixed encoding issue Feed: added images centering; added BackColor and ForeColor change MainFrame: added BackColor, ForeColor, and BackgroungImage change; added 'UpdateLogButton' when load completed ListImagesLoader: fixed wrong notification when no users found SettingsCLS: updated users loading algo
This commit is contained in:
Andy
@@ -11,17 +11,22 @@ Namespace API.Twitter
|
||||
Friend Property GifsDownload As Boolean
|
||||
Friend Property GifsSpecialFolder As String
|
||||
Friend Property GifsPrefix As String
|
||||
Friend Property UseMD5Comparison As Boolean = False
|
||||
Friend Property RemoveExistingDuplicates As Boolean = False
|
||||
Friend Sub New()
|
||||
End Sub
|
||||
Friend Sub New(ByVal s As SiteSettings)
|
||||
GifsDownload = s.GifsDownload.Value
|
||||
GifsSpecialFolder = s.GifsSpecialFolder.Value
|
||||
GifsPrefix = s.GifsPrefix.Value
|
||||
UseMD5Comparison = s.UseMD5Comparison.Value
|
||||
End Sub
|
||||
Friend Sub New(ByVal u As UserData)
|
||||
GifsDownload = u.GifsDownload
|
||||
GifsSpecialFolder = u.GifsSpecialFolder
|
||||
GifsPrefix = u.GifsPrefix
|
||||
UseMD5Comparison = u.UseMD5Comparison
|
||||
RemoveExistingDuplicates = u.RemoveExistingDuplicates
|
||||
End Sub
|
||||
End Class
|
||||
End Namespace
|
||||
73
SCrawler/API/Twitter/OptionsForm.Designer.vb
generated
73
SCrawler/API/Twitter/OptionsForm.Designer.vb
generated
@@ -22,16 +22,21 @@ Namespace API.Twitter
|
||||
Private components As System.ComponentModel.IContainer
|
||||
<System.Diagnostics.DebuggerStepThrough()>
|
||||
Private Sub InitializeComponent()
|
||||
Me.components = New System.ComponentModel.Container()
|
||||
Dim CONTAINER_MAIN As System.Windows.Forms.ToolStripContainer
|
||||
Dim TP_MAIN As System.Windows.Forms.TableLayoutPanel
|
||||
Dim ActionButton1 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
|
||||
Dim ActionButton3 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
|
||||
Dim resources As System.ComponentModel.ComponentResourceManager = New System.ComponentModel.ComponentResourceManager(GetType(OptionsForm))
|
||||
Dim ActionButton2 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
|
||||
Dim ActionButton4 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
|
||||
Dim TT_MAIN As System.Windows.Forms.ToolTip
|
||||
Me.CH_DOWN_GIFS = New System.Windows.Forms.CheckBox()
|
||||
Me.TXT_GIF_FOLDER = New PersonalUtilities.Forms.Controls.TextBoxExtended()
|
||||
Me.TXT_GIF_PREFIX = New PersonalUtilities.Forms.Controls.TextBoxExtended()
|
||||
Me.CH_USE_MD5 = New System.Windows.Forms.CheckBox()
|
||||
Me.CH_REMOVE_EXISTING_DUP = New System.Windows.Forms.CheckBox()
|
||||
CONTAINER_MAIN = New System.Windows.Forms.ToolStripContainer()
|
||||
TP_MAIN = New System.Windows.Forms.TableLayoutPanel()
|
||||
TT_MAIN = New System.Windows.Forms.ToolTip(Me.components)
|
||||
CONTAINER_MAIN.ContentPanel.SuspendLayout()
|
||||
CONTAINER_MAIN.SuspendLayout()
|
||||
TP_MAIN.SuspendLayout()
|
||||
@@ -45,13 +50,13 @@ Namespace API.Twitter
|
||||
'CONTAINER_MAIN.ContentPanel
|
||||
'
|
||||
CONTAINER_MAIN.ContentPanel.Controls.Add(TP_MAIN)
|
||||
CONTAINER_MAIN.ContentPanel.Size = New System.Drawing.Size(304, 84)
|
||||
CONTAINER_MAIN.ContentPanel.Size = New System.Drawing.Size(304, 161)
|
||||
CONTAINER_MAIN.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
CONTAINER_MAIN.LeftToolStripPanelVisible = False
|
||||
CONTAINER_MAIN.Location = New System.Drawing.Point(0, 0)
|
||||
CONTAINER_MAIN.Name = "CONTAINER_MAIN"
|
||||
CONTAINER_MAIN.RightToolStripPanelVisible = False
|
||||
CONTAINER_MAIN.Size = New System.Drawing.Size(304, 109)
|
||||
CONTAINER_MAIN.Size = New System.Drawing.Size(304, 161)
|
||||
CONTAINER_MAIN.TabIndex = 0
|
||||
CONTAINER_MAIN.TopToolStripPanelVisible = False
|
||||
'
|
||||
@@ -60,19 +65,22 @@ Namespace API.Twitter
|
||||
TP_MAIN.CellBorderStyle = System.Windows.Forms.TableLayoutPanelCellBorderStyle.[Single]
|
||||
TP_MAIN.ColumnCount = 1
|
||||
TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
|
||||
TP_MAIN.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Absolute, 20.0!))
|
||||
TP_MAIN.Controls.Add(Me.CH_DOWN_GIFS, 0, 0)
|
||||
TP_MAIN.Controls.Add(Me.TXT_GIF_FOLDER, 0, 1)
|
||||
TP_MAIN.Controls.Add(Me.TXT_GIF_PREFIX, 0, 2)
|
||||
TP_MAIN.Controls.Add(Me.CH_USE_MD5, 0, 3)
|
||||
TP_MAIN.Controls.Add(Me.CH_REMOVE_EXISTING_DUP, 0, 4)
|
||||
TP_MAIN.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
TP_MAIN.Location = New System.Drawing.Point(0, 0)
|
||||
TP_MAIN.Name = "TP_MAIN"
|
||||
TP_MAIN.RowCount = 4
|
||||
TP_MAIN.RowCount = 6
|
||||
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
||||
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
|
||||
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
|
||||
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
||||
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
||||
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
|
||||
TP_MAIN.Size = New System.Drawing.Size(304, 84)
|
||||
TP_MAIN.Size = New System.Drawing.Size(304, 161)
|
||||
TP_MAIN.TabIndex = 0
|
||||
'
|
||||
'CH_DOWN_GIFS
|
||||
@@ -89,10 +97,10 @@ Namespace API.Twitter
|
||||
'
|
||||
'TXT_GIF_FOLDER
|
||||
'
|
||||
ActionButton1.BackgroundImage = CType(resources.GetObject("ActionButton1.BackgroundImage"), System.Drawing.Image)
|
||||
ActionButton1.Name = "Clear"
|
||||
ActionButton1.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
|
||||
Me.TXT_GIF_FOLDER.Buttons.Add(ActionButton1)
|
||||
ActionButton3.BackgroundImage = CType(resources.GetObject("ActionButton3.BackgroundImage"), System.Drawing.Image)
|
||||
ActionButton3.Name = "Clear"
|
||||
ActionButton3.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
|
||||
Me.TXT_GIF_FOLDER.Buttons.Add(ActionButton3)
|
||||
Me.TXT_GIF_FOLDER.CaptionText = "GIFs special folder"
|
||||
Me.TXT_GIF_FOLDER.CaptionToolTipText = "Put the GIFs in a special folder"
|
||||
Me.TXT_GIF_FOLDER.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
@@ -103,10 +111,10 @@ Namespace API.Twitter
|
||||
'
|
||||
'TXT_GIF_PREFIX
|
||||
'
|
||||
ActionButton2.BackgroundImage = CType(resources.GetObject("ActionButton2.BackgroundImage"), System.Drawing.Image)
|
||||
ActionButton2.Name = "Clear"
|
||||
ActionButton2.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
|
||||
Me.TXT_GIF_PREFIX.Buttons.Add(ActionButton2)
|
||||
ActionButton4.BackgroundImage = CType(resources.GetObject("ActionButton4.BackgroundImage"), System.Drawing.Image)
|
||||
ActionButton4.Name = "Clear"
|
||||
ActionButton4.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.Clear
|
||||
Me.TXT_GIF_PREFIX.Buttons.Add(ActionButton4)
|
||||
Me.TXT_GIF_PREFIX.CaptionText = "GIF prefix"
|
||||
Me.TXT_GIF_PREFIX.CaptionToolTipText = "This prefix will be added to the beginning of the filename"
|
||||
Me.TXT_GIF_PREFIX.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
@@ -115,18 +123,45 @@ Namespace API.Twitter
|
||||
Me.TXT_GIF_PREFIX.Size = New System.Drawing.Size(296, 22)
|
||||
Me.TXT_GIF_PREFIX.TabIndex = 2
|
||||
'
|
||||
'CH_USE_MD5
|
||||
'
|
||||
Me.CH_USE_MD5.AutoSize = True
|
||||
Me.CH_USE_MD5.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
Me.CH_USE_MD5.Location = New System.Drawing.Point(4, 88)
|
||||
Me.CH_USE_MD5.Name = "CH_USE_MD5"
|
||||
Me.CH_USE_MD5.Padding = New System.Windows.Forms.Padding(100, 0, 0, 0)
|
||||
Me.CH_USE_MD5.Size = New System.Drawing.Size(296, 19)
|
||||
Me.CH_USE_MD5.TabIndex = 3
|
||||
Me.CH_USE_MD5.Text = "Use MD5 comparison"
|
||||
TT_MAIN.SetToolTip(Me.CH_USE_MD5, "Each image will be checked for existence using MD5")
|
||||
Me.CH_USE_MD5.UseVisualStyleBackColor = True
|
||||
'
|
||||
'CH_REMOVE_EXISTING_DUP
|
||||
'
|
||||
Me.CH_REMOVE_EXISTING_DUP.AutoSize = True
|
||||
Me.CH_REMOVE_EXISTING_DUP.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
Me.CH_REMOVE_EXISTING_DUP.Location = New System.Drawing.Point(4, 114)
|
||||
Me.CH_REMOVE_EXISTING_DUP.Name = "CH_REMOVE_EXISTING_DUP"
|
||||
Me.CH_REMOVE_EXISTING_DUP.Padding = New System.Windows.Forms.Padding(100, 0, 0, 0)
|
||||
Me.CH_REMOVE_EXISTING_DUP.Size = New System.Drawing.Size(296, 19)
|
||||
Me.CH_REMOVE_EXISTING_DUP.TabIndex = 4
|
||||
Me.CH_REMOVE_EXISTING_DUP.Text = "Remove existing duplicates"
|
||||
TT_MAIN.SetToolTip(Me.CH_REMOVE_EXISTING_DUP, "Existing files will be checked for duplicates and duplicates removed." & Global.Microsoft.VisualBasic.ChrW(13) & Global.Microsoft.VisualBasic.ChrW(10) & "Works only" &
|
||||
" on the first activation 'Use MD5 comparison'.")
|
||||
Me.CH_REMOVE_EXISTING_DUP.UseVisualStyleBackColor = True
|
||||
'
|
||||
'OptionsForm
|
||||
'
|
||||
Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)
|
||||
Me.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font
|
||||
Me.ClientSize = New System.Drawing.Size(304, 109)
|
||||
Me.ClientSize = New System.Drawing.Size(304, 161)
|
||||
Me.Controls.Add(CONTAINER_MAIN)
|
||||
Me.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedSingle
|
||||
Me.Icon = Global.SCrawler.My.Resources.SiteResources.TwitterIcon_32
|
||||
Me.MaximizeBox = False
|
||||
Me.MaximumSize = New System.Drawing.Size(320, 148)
|
||||
Me.MaximumSize = New System.Drawing.Size(320, 200)
|
||||
Me.MinimizeBox = False
|
||||
Me.MinimumSize = New System.Drawing.Size(320, 148)
|
||||
Me.MinimumSize = New System.Drawing.Size(320, 200)
|
||||
Me.Name = "OptionsForm"
|
||||
Me.ShowInTaskbar = False
|
||||
Me.SizeGripStyle = System.Windows.Forms.SizeGripStyle.Hide
|
||||
@@ -144,5 +179,7 @@ Namespace API.Twitter
|
||||
Private WithEvents CH_DOWN_GIFS As CheckBox
|
||||
Private WithEvents TXT_GIF_FOLDER As PersonalUtilities.Forms.Controls.TextBoxExtended
|
||||
Private WithEvents TXT_GIF_PREFIX As PersonalUtilities.Forms.Controls.TextBoxExtended
|
||||
Private WithEvents CH_USE_MD5 As CheckBox
|
||||
Private WithEvents CH_REMOVE_EXISTING_DUP As CheckBox
|
||||
End Class
|
||||
End Namespace
|
||||
@@ -124,7 +124,7 @@
|
||||
<value>False</value>
|
||||
</metadata>
|
||||
<assembly alias="System.Drawing" name="System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a" />
|
||||
<data name="ActionButton1.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
|
||||
<data name="ActionButton3.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
|
||||
<value>
|
||||
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
|
||||
xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go
|
||||
@@ -132,7 +132,7 @@
|
||||
AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC
|
||||
</value>
|
||||
</data>
|
||||
<data name="ActionButton2.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
|
||||
<data name="ActionButton4.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
|
||||
<value>
|
||||
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
|
||||
xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go
|
||||
@@ -140,4 +140,16 @@
|
||||
AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC
|
||||
</value>
|
||||
</data>
|
||||
<metadata name="TT_MAIN.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
|
||||
<value>False</value>
|
||||
</metadata>
|
||||
<metadata name="TT_MAIN.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||
<value>17, 17</value>
|
||||
</metadata>
|
||||
<metadata name="TT_MAIN.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
|
||||
<value>False</value>
|
||||
</metadata>
|
||||
<metadata name="TT_MAIN.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||
<value>17, 17</value>
|
||||
</metadata>
|
||||
</root>
|
||||
@@ -30,6 +30,8 @@ Namespace API.Twitter
|
||||
TXT_GIF_FOLDER.Tag = NameOf(SiteSettings.GifsSpecialFolder)
|
||||
TXT_GIF_PREFIX.Text = .GifsPrefix
|
||||
TXT_GIF_PREFIX.Tag = NameOf(SiteSettings.GifsPrefix)
|
||||
CH_USE_MD5.Checked = .UseMD5Comparison
|
||||
CH_REMOVE_EXISTING_DUP.Checked = .RemoveExistingDuplicates
|
||||
|
||||
Try
|
||||
Dim p As PropertyOption
|
||||
@@ -59,6 +61,8 @@ Namespace API.Twitter
|
||||
.GifsDownload = CH_DOWN_GIFS.Checked
|
||||
.GifsSpecialFolder = TXT_GIF_FOLDER.Text
|
||||
.GifsPrefix = TXT_GIF_PREFIX.Text
|
||||
.UseMD5Comparison = CH_USE_MD5.Checked
|
||||
.RemoveExistingDuplicates = CH_REMOVE_EXISTING_DUP.Checked
|
||||
End With
|
||||
MyDefs.CloseForm()
|
||||
End Sub
|
||||
|
||||
@@ -69,6 +69,8 @@ Namespace API.Twitter
|
||||
Throw New NotImplementedException("[GetFormat] is not available in the context of [TimersChecker]")
|
||||
End Function
|
||||
End Class
|
||||
<PropertyOption(IsAuth:=False, ControlText:="Use MD5 comparison", ControlToolTip:="Each image will be checked for existence using MD5"), PXML>
|
||||
Friend ReadOnly Property UseMD5Comparison As PropertyValue
|
||||
#End Region
|
||||
Friend Overrides ReadOnly Property Responser As Responser
|
||||
#End Region
|
||||
@@ -115,6 +117,7 @@ Namespace API.Twitter
|
||||
GifsSpecialFolder = New PropertyValue(String.Empty, GetType(String))
|
||||
GifsPrefix = New PropertyValue("GIF_")
|
||||
GifStringChecker = New GifStringProvider
|
||||
UseMD5Comparison = New PropertyValue(False)
|
||||
|
||||
UserRegex = RParams.DMS("[htps:/]{7,8}.*?twitter.com/([^/]+)", 1)
|
||||
UrlPatternUser = "https://twitter.com/{0}"
|
||||
|
||||
@@ -7,13 +7,16 @@
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports System.Net
|
||||
Imports System.Drawing
|
||||
Imports System.Threading
|
||||
Imports SCrawler.API.Base
|
||||
Imports PersonalUtilities.Functions.XML
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Imports PersonalUtilities.Tools.Web.Clients
|
||||
Imports PersonalUtilities.Tools.Web.Documents.JSON
|
||||
Imports PersonalUtilities.Tools.ImageRenderer
|
||||
Imports UStates = SCrawler.API.Base.UserMedia.States
|
||||
Imports UTypes = SCrawler.API.Base.UserMedia.Types
|
||||
Namespace API.Twitter
|
||||
Friend Class UserData : Inherits UserDataBase
|
||||
Private Const SinglePostUrl As String = "https://api.twitter.com/1.1/statuses/show.json?id={0}&tweet_mode=extended"
|
||||
@@ -21,12 +24,18 @@ Namespace API.Twitter
|
||||
Private Const Name_GifsDownload As String = "GifsDownload"
|
||||
Private Const Name_GifsSpecialFolder As String = "GifsSpecialFolder"
|
||||
Private Const Name_GifsPrefix As String = "GifsPrefix"
|
||||
Private Const Name_UseMD5Comparison As String = "UseMD5Comparison"
|
||||
Private Const Name_RemoveExistingDuplicates As String = "RemoveExistingDuplicates"
|
||||
Private Const Name_StartMD5Checked As String = "StartMD5Checked"
|
||||
#End Region
|
||||
#Region "Declarations"
|
||||
Friend Property GifsDownload As Boolean
|
||||
Friend Property GifsSpecialFolder As String
|
||||
Friend Property GifsPrefix As String
|
||||
Private ReadOnly _DataNames As List(Of String)
|
||||
Friend Property UseMD5Comparison As Boolean = False
|
||||
Private StartMD5Checked As Boolean = False
|
||||
Friend Property RemoveExistingDuplicates As Boolean = False
|
||||
#End Region
|
||||
#Region "Exchange options"
|
||||
Friend Overrides Function ExchangeOptionsGet() As Object
|
||||
@@ -38,6 +47,8 @@ Namespace API.Twitter
|
||||
GifsDownload = .GifsDownload
|
||||
GifsSpecialFolder = .GifsSpecialFolder
|
||||
GifsPrefix = .GifsPrefix
|
||||
UseMD5Comparison = .UseMD5Comparison
|
||||
RemoveExistingDuplicates = .RemoveExistingDuplicates
|
||||
End With
|
||||
End If
|
||||
End Sub
|
||||
@@ -55,10 +66,16 @@ Namespace API.Twitter
|
||||
Else
|
||||
GifsPrefix = Container.Value(Name_GifsPrefix)
|
||||
End If
|
||||
UseMD5Comparison = Container.Value(Name_UseMD5Comparison).FromXML(Of Boolean)(False)
|
||||
RemoveExistingDuplicates = Container.Value(Name_RemoveExistingDuplicates).FromXML(Of Boolean)(False)
|
||||
StartMD5Checked = Container.Value(Name_StartMD5Checked).FromXML(Of Boolean)(False)
|
||||
Else
|
||||
Container.Add(Name_GifsDownload, GifsDownload.BoolToInteger)
|
||||
Container.Add(Name_GifsSpecialFolder, GifsSpecialFolder)
|
||||
Container.Add(Name_GifsPrefix, GifsPrefix)
|
||||
Container.Add(Name_UseMD5Comparison, UseMD5Comparison.BoolToInteger)
|
||||
Container.Add(Name_RemoveExistingDuplicates, RemoveExistingDuplicates.BoolToInteger)
|
||||
Container.Add(Name_StartMD5Checked, StartMD5Checked.BoolToInteger)
|
||||
End If
|
||||
End Sub
|
||||
#End Region
|
||||
@@ -70,6 +87,7 @@ Namespace API.Twitter
|
||||
Else
|
||||
If _ContentList.Count > 0 Then _DataNames.ListAddList(_ContentList.Select(Function(c) c.File.File), LAP.ClearBeforeAdd, LAP.NotContainsOnly)
|
||||
DownloadData(String.Empty, Token)
|
||||
If UseMD5Comparison Then ValidateMD5(Token)
|
||||
End If
|
||||
End Sub
|
||||
Private Overloads Sub DownloadData(ByVal POST As String, ByVal Token As CancellationToken)
|
||||
@@ -100,6 +118,30 @@ Namespace API.Twitter
|
||||
If Not r.IsEmptyString Then
|
||||
Using w As EContainer = JsonDocument.Parse(r)
|
||||
If w.ListExists Then
|
||||
|
||||
If Not IsSavedPosts And POST.IsEmptyString And Not w.ItemF({0, "user"}) Is Nothing Then
|
||||
With w.ItemF({0, "user"})
|
||||
If .Value("screen_name").StringToLower = Name Then
|
||||
UserSiteNameUpdate(.Value("name"))
|
||||
UserDescriptionUpdate(.Value("description"))
|
||||
Dim __getImage As Action(Of String) = Sub(ByVal img As String)
|
||||
If Not img.IsEmptyString Then
|
||||
Dim __imgFile As SFile = UrlFile(img, True)
|
||||
If Not __imgFile.Name.IsEmptyString Then
|
||||
If __imgFile.Extension.IsEmptyString Then __imgFile.Extension = "jpg"
|
||||
__imgFile.Path = MyFile.CutPath.Path
|
||||
If Not __imgFile.Exists Then GetWebFile(img, __imgFile, EDP.None)
|
||||
End If
|
||||
End If
|
||||
End Sub
|
||||
Dim icon$ = .Value("profile_image_url_https")
|
||||
If Not icon.IsEmptyString Then icon = icon.Replace("_normal", String.Empty)
|
||||
__getImage.Invoke(.Value("profile_banner_url"))
|
||||
__getImage.Invoke(icon)
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
|
||||
For Each nn In If(IsSavedPosts, w({"globalObjects", "tweets"}).XmlIfNothing, w)
|
||||
ThrowAny(Token)
|
||||
If nn.Count > 0 Then
|
||||
@@ -114,9 +156,6 @@ Namespace API.Twitter
|
||||
End If
|
||||
End If
|
||||
|
||||
If Not IsSavedPosts AndAlso UserDescriptionNeedToUpdate() AndAlso nn.Value({"user"}, "screen_name") = Name Then _
|
||||
UserDescriptionUpdate(nn.Value({"user"}, "description"))
|
||||
|
||||
'Date Pattern:
|
||||
'Sat Jan 01 01:10:15 +0000 2000
|
||||
If nn.Contains("created_at") Then PostDate = nn("created_at").Value Else PostDate = String.Empty
|
||||
@@ -174,7 +213,7 @@ Namespace API.Twitter
|
||||
If Not dName.IsEmptyString AndAlso Not _DataNames.Contains(dName) Then
|
||||
_DataNames.Add(dName)
|
||||
_TempMediaList.ListAddValue(MediaFromData(m("media_url").Value,
|
||||
PostID, PostDate, GetPictureOption(m), State), LNC)
|
||||
PostID, PostDate, GetPictureOption(m), State, UTypes.Picture), LNC)
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
@@ -190,7 +229,7 @@ Namespace API.Twitter
|
||||
Dim f$ = UrlFile(URL)
|
||||
If Not f.IsEmptyString AndAlso Not _DataNames.Contains(f) Then
|
||||
_DataNames.Add(f)
|
||||
_TempMediaList.ListAddValue(MediaFromData(URL, PostID, PostDate,, State), LNC)
|
||||
_TempMediaList.ListAddValue(MediaFromData(URL, PostID, PostDate,, State, UTypes.Video), LNC)
|
||||
End If
|
||||
Return True
|
||||
End If
|
||||
@@ -218,7 +257,7 @@ Namespace API.Twitter
|
||||
ff = UrlFile(url)
|
||||
If Not ff.IsEmptyString Then
|
||||
If GifsDownload And Not _DataNames.Contains(ff) Then
|
||||
m = MediaFromData(url, PostID, PostDate,, State)
|
||||
m = MediaFromData(url, PostID, PostDate,, State, UTypes.Video)
|
||||
f = m.File
|
||||
If Not f.IsEmptyString And Not GifsPrefix.IsEmptyString Then f.Name = $"{GifsPrefix}{f.Name}" : m.File = f
|
||||
If Not GifsSpecialFolder.IsEmptyString Then m.SpecialFolder = $"{GifsSpecialFolder}*"
|
||||
@@ -298,6 +337,131 @@ Namespace API.Twitter
|
||||
End Try
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "MD5 support"
|
||||
Private Const VALIDATE_MD5_ERROR As String = "VALIDATE_MD5_ERROR"
|
||||
Private Sub ValidateMD5(ByVal Token As CancellationToken)
|
||||
Try
|
||||
Dim missingMD5 As Predicate(Of UserMedia) = Function(d) (d.Type = UTypes.GIF Or d.Type = UTypes.Picture) And d.MD5.IsEmptyString
|
||||
If UseMD5Comparison And _TempMediaList.Exists(missingMD5) Then
|
||||
Dim i%
|
||||
Dim data As UserMedia = Nothing
|
||||
Dim hashList As New Dictionary(Of String, SFile)
|
||||
Dim f As SFile
|
||||
Dim ErrMD5 As New ErrorsDescriber(EDP.ReturnValue)
|
||||
Dim __getMD5 As Func(Of UserMedia, Boolean, String) =
|
||||
Function(ByVal __data As UserMedia, ByVal IsUrl As Boolean) As String
|
||||
Try
|
||||
Dim ImgFormat As Imaging.ImageFormat = Nothing
|
||||
Dim hash$ = String.Empty
|
||||
Dim __isGif As Boolean = False
|
||||
If __data.Type = UTypes.GIF Then
|
||||
ImgFormat = Imaging.ImageFormat.Gif
|
||||
__isGif = True
|
||||
ElseIf Not __data.File.IsEmptyString Then
|
||||
ImgFormat = GetImageFormat(__data.File)
|
||||
End If
|
||||
If ImgFormat Is Nothing Then ImgFormat = Imaging.ImageFormat.Jpeg
|
||||
If IsUrl Then
|
||||
hash = ByteArrayToString(GetMD5(SFile.GetBytesFromNet(__data.URL_BASE.IfNullOrEmpty(__data.URL), ErrMD5), ImgFormat, ErrMD5))
|
||||
Else
|
||||
hash = ByteArrayToString(GetMD5(SFile.GetBytes(__data.File, ErrMD5), ImgFormat, ErrMD5))
|
||||
End If
|
||||
If hash.IsEmptyString And Not __isGif Then
|
||||
If ImgFormat Is Imaging.ImageFormat.Jpeg Then ImgFormat = Imaging.ImageFormat.Png Else ImgFormat = Imaging.ImageFormat.Jpeg
|
||||
If IsUrl Then
|
||||
hash = ByteArrayToString(GetMD5(SFile.GetBytesFromNet(__data.URL_BASE.IfNullOrEmpty(__data.URL), ErrMD5), ImgFormat, ErrMD5))
|
||||
Else
|
||||
hash = ByteArrayToString(GetMD5(SFile.GetBytes(__data.File, ErrMD5), ImgFormat, ErrMD5))
|
||||
End If
|
||||
End If
|
||||
Return hash
|
||||
Catch
|
||||
Return String.Empty
|
||||
End Try
|
||||
End Function
|
||||
If Not StartMD5Checked Then
|
||||
StartMD5Checked = True
|
||||
If _ContentList.Exists(missingMD5) Then
|
||||
Dim existingFiles As List(Of SFile) = SFile.GetFiles(MyFileSettings.CutPath, "*.jpg|*.jpeg|*.png|*.gif",, EDP.ReturnValue).ListIfNothing
|
||||
Dim eIndx%
|
||||
Dim eFinder As Predicate(Of SFile) = Function(ff) ff.File = data.File.File
|
||||
If RemoveExistingDuplicates Then
|
||||
RemoveExistingDuplicates = False
|
||||
_ForceSaveUserInfo = True
|
||||
If existingFiles.Count > 0 Then
|
||||
Dim h$
|
||||
For i = existingFiles.Count - 1 To 0 Step -1
|
||||
h = __getMD5(New UserMedia With {.File = existingFiles(i)}, False)
|
||||
If Not h.IsEmptyString Then
|
||||
If hashList.ContainsKey(h) Then
|
||||
MyMainLOG = $"[{ToStringForLog()}]: Removed image [{existingFiles(i).File}] (duplicate of [{hashList(h).File}])"
|
||||
existingFiles(i).Delete(SFO.File, SFODelete.DeleteToRecycleBin, ErrMD5)
|
||||
existingFiles.RemoveAt(i)
|
||||
Else
|
||||
hashList.Add(h, existingFiles(i))
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
End If
|
||||
For i = 0 To _ContentList.Count - 1
|
||||
data = _ContentList(i)
|
||||
If (data.Type = UTypes.GIF Or data.Type = UTypes.Picture) Then
|
||||
If data.MD5.IsEmptyString Then
|
||||
ThrowAny(Token)
|
||||
eIndx = existingFiles.FindIndex(eFinder)
|
||||
If eIndx >= 0 Then
|
||||
data.MD5 = __getMD5(New UserMedia With {.File = existingFiles(eIndx)}, False)
|
||||
If Not data.MD5.IsEmptyString Then _ContentList(i) = data : _ForceSaveUserData = True
|
||||
End If
|
||||
End If
|
||||
existingFiles.RemoveAll(eFinder)
|
||||
End If
|
||||
Next
|
||||
If existingFiles.Count > 0 Then
|
||||
For i = 0 To existingFiles.Count - 1
|
||||
f = existingFiles(i)
|
||||
data = New UserMedia(f.File) With {
|
||||
.State = UStates.Downloaded,
|
||||
.Type = IIf(f.Extension = "gif", UTypes.GIF, UTypes.Picture),
|
||||
.File = f
|
||||
}
|
||||
ThrowAny(Token)
|
||||
data.MD5 = __getMD5(data, False)
|
||||
If Not data.MD5.IsEmptyString Then _ContentList.Add(data) : _ForceSaveUserData = True
|
||||
Next
|
||||
existingFiles.Clear()
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
|
||||
If _ContentList.Count > 0 Then
|
||||
With _ContentList.Select(Function(d) d.MD5)
|
||||
If .ListExists Then .ToList.ForEach(Sub(md5value) If Not hashList.ContainsKey(md5value) Then hashList.Add(md5value, New SFile))
|
||||
End With
|
||||
End If
|
||||
|
||||
For i = _TempMediaList.Count - 1 To 0 Step -1
|
||||
data = _TempMediaList(i)
|
||||
If missingMD5(data) Then
|
||||
ThrowAny(Token)
|
||||
data.MD5 = __getMD5(data, True)
|
||||
If Not data.MD5.IsEmptyString Then
|
||||
If hashList.ContainsKey(data.MD5) Then
|
||||
_TempMediaList.RemoveAt(i)
|
||||
Else
|
||||
hashList.Add(data.MD5, New SFile)
|
||||
_TempMediaList(i) = data
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, "ValidateMD5",, VALIDATE_MD5_ERROR)
|
||||
End Try
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Get video static"
|
||||
Friend Shared Function GetVideoInfo(ByVal URL As String, ByVal resp As Responser) As IEnumerable(Of UserMedia)
|
||||
Try
|
||||
@@ -310,7 +474,7 @@ Namespace API.Twitter
|
||||
Using j As EContainer = JsonDocument.Parse(r)
|
||||
If j.ListExists Then
|
||||
Dim u$ = GetVideoNodeURL(j)
|
||||
If Not u.IsEmptyString Then Return {MediaFromData(u, PostID, String.Empty)}
|
||||
If Not u.IsEmptyString Then Return {MediaFromData(u, PostID, String.Empty,,, UTypes.Video)}
|
||||
End If
|
||||
End Using
|
||||
End If
|
||||
@@ -359,10 +523,17 @@ Namespace API.Twitter
|
||||
End Function
|
||||
#End Region
|
||||
#Region "UrlFile"
|
||||
Private Function UrlFile(ByVal URL As String) As String
|
||||
Private Function UrlFile(ByVal URL As String, Optional ByVal GetWithoutExtension As Boolean = False) As String
|
||||
Try
|
||||
Dim f As SFile = CStr(RegexReplace(LinkFormatterSecure(RegexReplace(URL.Replace("\", String.Empty), LinkPattern)), FilesPattern))
|
||||
If Not f.IsEmptyString Then Return f.File Else Return String.Empty
|
||||
If Not URL.IsEmptyString Then
|
||||
Dim f As SFile = CStr(RegexReplace(LinkFormatterSecure(RegexReplace(URL.Replace("\", String.Empty), LinkPattern)), FilesPattern))
|
||||
If f.IsEmptyString And GetWithoutExtension Then
|
||||
URL = LinkFormatterSecure(RegexReplace(URL.Replace("\", String.Empty), LinkPattern))
|
||||
If Not URL.IsEmptyString Then f = New SFile With {.Name = URL.Split("/").LastOrDefault}
|
||||
End If
|
||||
If Not f.IsEmptyString Then Return f.File
|
||||
End If
|
||||
Return String.Empty
|
||||
Catch ex As Exception
|
||||
Return String.Empty
|
||||
End Try
|
||||
@@ -371,9 +542,10 @@ Namespace API.Twitter
|
||||
#Region "Create media"
|
||||
Private Shared Function MediaFromData(ByVal _URL As String, ByVal PostID As String, ByVal PostDate As String,
|
||||
Optional ByVal _PictureOption As String = Nothing,
|
||||
Optional ByVal State As UStates = UStates.Unknown) As UserMedia
|
||||
Optional ByVal State As UStates = UStates.Unknown,
|
||||
Optional ByVal Type As UTypes = UTypes.Undefined) As UserMedia
|
||||
_URL = LinkFormatterSecure(RegexReplace(_URL.Replace("\", String.Empty), LinkPattern))
|
||||
Dim m As New UserMedia(_URL) With {.PictureOption = _PictureOption, .Post = New UserPost With {.ID = PostID}}
|
||||
Dim m As New UserMedia(_URL) With {.PictureOption = _PictureOption, .Post = New UserPost With {.ID = PostID}, .Type = Type}
|
||||
If Not m.URL.IsEmptyString Then m.File = CStr(RegexReplace(m.URL, FilesPattern))
|
||||
If Not m.PictureOption.IsEmptyString And Not m.File.IsEmptyString And Not m.URL.IsEmptyString Then
|
||||
m.URL = $"{m.URL.Replace($".{m.File.Extension}", String.Empty)}?format={m.File.Extension}&name={m.PictureOption}"
|
||||
@@ -391,20 +563,25 @@ Namespace API.Twitter
|
||||
#Region "Exception"
|
||||
Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False,
|
||||
Optional ByVal EObj As Object = Nothing) As Integer
|
||||
With Responser
|
||||
If .StatusCode = HttpStatusCode.NotFound Then
|
||||
UserExists = False
|
||||
ElseIf .StatusCode = HttpStatusCode.Unauthorized Then
|
||||
UserSuspended = True
|
||||
ElseIf .StatusCode = HttpStatusCode.BadRequest Then
|
||||
MyMainLOG = "Twitter has invalid credentials"
|
||||
ElseIf .StatusCode = HttpStatusCode.ServiceUnavailable Or .StatusCode = HttpStatusCode.InternalServerError Then
|
||||
MyMainLOG = $"[{CInt(.StatusCode)}] Twitter is currently unavailable ({ToString()})"
|
||||
Else
|
||||
If Not FromPE Then LogError(ex, Message) : HasError = True
|
||||
Return 0
|
||||
End If
|
||||
End With
|
||||
If AEquals(EObj, VALIDATE_MD5_ERROR) Then
|
||||
If Not FromPE Then LogError(ex, Message)
|
||||
Return 0
|
||||
Else
|
||||
With Responser
|
||||
If .StatusCode = HttpStatusCode.NotFound Then
|
||||
UserExists = False
|
||||
ElseIf .StatusCode = HttpStatusCode.Unauthorized Then
|
||||
UserSuspended = True
|
||||
ElseIf .StatusCode = HttpStatusCode.BadRequest Then
|
||||
MyMainLOG = "Twitter has invalid credentials"
|
||||
ElseIf .StatusCode = HttpStatusCode.ServiceUnavailable Or .StatusCode = HttpStatusCode.InternalServerError Then
|
||||
MyMainLOG = $"[{CInt(.StatusCode)}] Twitter is currently unavailable ({ToString()})"
|
||||
Else
|
||||
If Not FromPE Then LogError(ex, Message) : HasError = True
|
||||
Return 0
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
Return 1
|
||||
End Function
|
||||
#End Region
|
||||
|
||||
Reference in New Issue
Block a user