geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-15 00:02:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

2.0.0.3

Browse Source 
Added GetUserMediaOnly for Reddit
Fixed Reddit CrossPosts issue
Fixed validating new users (collections)
Fixed incorrect count of Instagram download tasks
This commit is contained in:
Andy
2022-02-02 07:34:36 +03:00
parent 2a2fc45a91
commit 5d5ce29f36
10 changed files with 115 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
SCrawler/API/Base/SiteSettings.vb
View File

@@ -123,7 +123,6 @@ Namespace API.Base
Dim n() As String = {SettingsCLS.Name_Node_Sites, Site.ToString}
_Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"), _XML, n, XMLValue(Of SFile).ToFilePath)
_Path.ReplaceByValue("Path", {Site.ToString})
_XML.Remove(Site.ToString)
Temporary = New XMLValue(Of Boolean)
@@ -138,12 +137,7 @@ Namespace API.Base
DownloadVideos.SetExtended("DownloadVideos", True, _XML, n)
DownloadVideos.SetDefault(_Vids)
If Site = Sites.Twitter Then
GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n)
GetUserMediaOnly.ReplaceByValue("TwitterDefaultGetUserMedia", n)
Else
GetUserMediaOnly = New XMLValue(Of Boolean)
End If
GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n)
CreateProp(InstaHashUpdateRequired, Sites.Instagram, "InstaHashUpdateRequired", True, _XML, n)
CreateProp(InstaHash, Sites.Instagram, "InstaHash", String.Empty, _XML, n)

59
SCrawler/API/Reddit/UserData.vb
View File

@@ -54,12 +54,14 @@ Namespace API.Reddit
Private Sub New()
ChannelPostsNames = New List(Of String)
_ExistsUsersNames = New List(Of String)
_CrossPosts = New List(Of String)
End Sub
''' <summary>Default initializer</summary>
Friend Sub New(ByVal u As UserInfo, Optional ByVal _LoadUserInformation As Boolean = True, Optional ByVal InvokeImageHandler As Boolean = True)
MyBase.New(InvokeImageHandler)
ChannelPostsNames = New List(Of String)
_ExistsUsersNames = New List(Of String)
_CrossPosts = New List(Of String)
User = u
If _LoadUserInformation Then LoadUserInformation()
End Sub
@@ -108,10 +110,13 @@ Namespace API.Reddit
#End Region
#Region "Download Functions (User, Channel)"
Private _TotalPostsDownloaded As Integer = 0
Private ReadOnly _CrossPosts As List(Of String)
Private Sub DownloadDataUser(ByVal POST As String, ByVal Token As CancellationToken)
Const CPRI$ = "crosspostRootId"
Const CPPI$ = "crosspostParentId"
Dim URL$ = String.Empty
Try
Dim PostID$ = String.Empty
Dim PostID$ = String.Empty, PostTmp$ = String.Empty
Dim PostDate$
Dim n As EContainer, nn As EContainer, s As EContainer
Dim NewPostDetected As Boolean = False
@@ -120,8 +125,10 @@ Namespace API.Reddit
Dim added As Boolean
Dim __ItemType$
Dim tmpType As UTypes
Dim CheckNode As Predicate(Of EContainer) = Function(e) e("author").XmlIfNothingValue("/").ToLower.Equals(Name.ToLower)
Dim IsCrossPost As Predicate(Of EContainer) = Function(e) Not (e.Value(CPRI).IsEmptyString And e.Value(CPPI).IsEmptyString)
Dim CheckNode As Predicate(Of EContainer) = Function(e) Not ParseUserMediaOnly OrElse e("author").XmlIfNothingValue("/").ToLower.Equals(Name.ToLower)
Dim UPicType As Func(Of String, UTypes) = Function(input) IIf(input = "image", UTypes.Picture, UTypes.GIF)
Dim _PostID As Func(Of String) = Function() IIf(PostTmp.IsEmptyString, PostID, PostTmp)
URL = $"https://gateway.reddit.com/desktopapi/v1/user/{Name}/posts?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort=new&t=all&layout=classic"
ThrowAny(Token)
@@ -134,39 +141,51 @@ Namespace API.Reddit
For Each nn In n
ThrowAny(Token)
If nn.Count > 0 Then
PostID = nn.Name
If PostID.IsEmptyString AndAlso nn.Contains("id") Then PostID = nn("id").Value
If nn.Contains("created") Then PostDate = nn("created").Value Else PostDate = String.Empty
If Not _TempPostsList.Contains(PostID) Then
NewPostDetected = True
_TempPostsList.Add(PostID)
Else
ExistsDetected = True
Continue For
End If
If CheckNode(nn) Then
'Obtain post ID
PostTmp = nn.Name
If PostTmp.IsEmptyString Then PostTmp = nn.Value("id")
If PostTmp.IsEmptyString Then Continue For
'Check for CrossPost
If IsCrossPost(nn) Then
_CrossPosts.ListAddList({nn.Value(CPRI), nn.Value(CPPI)}, LNC)
Continue For
Else
If Not _CrossPosts.Contains(PostTmp) Then PostID = PostTmp : PostTmp = String.Empty
End If
'Download decision
If Not _TempPostsList.Contains(_PostID()) Then
NewPostDetected = True
_TempPostsList.Add(_PostID())
Else
If Not _CrossPosts.Contains(_PostID()) Then ExistsDetected = True
Continue For
End If
If nn.Contains("created") Then PostDate = nn("created").Value Else PostDate = String.Empty
_ItemsBefore = _TempMediaList.Count
added = True
s = nn.ItemF({"source", "url"})
If s.XmlIfNothingValue("/").Contains("redgifs.com") Then
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel), LNC)
ElseIf Not CreateImgurMedia(s.XmlIfNothingValue, PostID, PostDate,, IsChannel) Then
_TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, _PostID(), PostDate,, IsChannel), LNC)
ElseIf Not CreateImgurMedia(s.XmlIfNothingValue, _PostID(), PostDate,, IsChannel) Then
s = nn.ItemF({"media"}).XmlIfNothing
__ItemType = s("type").XmlIfNothingValue
Select Case __ItemType
Case "gallery" : If Not DownloadGallery(s, PostID, PostDate) Then added = False
Case "gallery" : If Not DownloadGallery(s, _PostID(), PostDate) Then added = False
Case "image", "gifvideo"
If s.Contains("content") Then
_TempMediaList.ListAddValue(MediaFromData(UPicType(__ItemType), s.Value("content"),
PostID, PostDate,, IsChannel), LNC)
_PostID(), PostDate,, IsChannel), LNC)
Else
added = False
End If
Case "video"
If Settings.UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then
_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"),
PostID, PostDate,, IsChannel), LNC)
_PostID(), PostDate,, IsChannel), LNC)
Else
added = False
End If
@@ -186,7 +205,7 @@ Namespace API.Reddit
End Select
End With
If Not tmpType = UTypes.Undefined Then
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel), LNC)
_TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, _PostID(), PostDate,, IsChannel), LNC)
End If
End If
End If
@@ -596,7 +615,7 @@ Namespace API.Reddit
Return 1
End Function
Protected Overrides Sub Dispose(ByVal disposing As Boolean)
If Not disposedValue And disposing Then ChannelPostsNames.Clear() : _ExistsUsersNames.Clear()
If Not disposedValue And disposing Then ChannelPostsNames.Clear() : _ExistsUsersNames.Clear() : _CrossPosts.Clear()
MyBase.Dispose(disposing)
End Sub
End Class

78
SCrawler/Editors/GlobalSettingsForm.Designer.vb generated
View File

@@ -53,6 +53,7 @@
Me.OPT_FILE_DATE_END = New System.Windows.Forms.RadioButton()
Me.CH_EXIT_CONFIRM = New System.Windows.Forms.CheckBox()
Me.CH_CLOSE_TO_TRAY = New System.Windows.Forms.CheckBox()
Me.CH_SHOW_NOTIFY = New System.Windows.Forms.CheckBox()
Me.CH_COPY_CHANNEL_USER_IMAGE = New System.Windows.Forms.CheckBox()
Me.CH_DEF_TEMP = New System.Windows.Forms.CheckBox()
Me.CH_DOWN_IMAGES = New System.Windows.Forms.CheckBox()
@@ -76,7 +77,7 @@
Me.TAB_DEFS_REDGIFS = New System.Windows.Forms.TabPage()
Me.DEFS_REDGIFS = New SCrawler.Editors.SiteDefaults()
Me.CONTAINER_MAIN = New System.Windows.Forms.ToolStripContainer()
Me.CH_SHOW_NOTIFY = New System.Windows.Forms.CheckBox()
Me.CH_REDDIT_USER_MEDIA = New System.Windows.Forms.CheckBox()
TP_BASIS = New System.Windows.Forms.TableLayoutPanel()
TP_IMAGES = New System.Windows.Forms.TableLayoutPanel()
TP_FILE_NAME = New System.Windows.Forms.TableLayoutPanel()
@@ -162,7 +163,7 @@
TP_BASIS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
TP_BASIS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
TP_BASIS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_BASIS.Size = New System.Drawing.Size(570, 341)
TP_BASIS.Size = New System.Drawing.Size(570, 366)
TP_BASIS.TabIndex = 0
'
'TXT_GLOBAL_PATH
@@ -469,6 +470,17 @@
Me.CH_CLOSE_TO_TRAY.Text = "Close to tray"
Me.CH_CLOSE_TO_TRAY.UseVisualStyleBackColor = True
'
'CH_SHOW_NOTIFY
'
Me.CH_SHOW_NOTIFY.AutoSize = True
Me.CH_SHOW_NOTIFY.Dock = System.Windows.Forms.DockStyle.Fill
Me.CH_SHOW_NOTIFY.Location = New System.Drawing.Point(4, 318)
Me.CH_SHOW_NOTIFY.Name = "CH_SHOW_NOTIFY"
Me.CH_SHOW_NOTIFY.Size = New System.Drawing.Size(562, 19)
Me.CH_SHOW_NOTIFY.TabIndex = 11
Me.CH_SHOW_NOTIFY.Text = "Show notifications"
Me.CH_SHOW_NOTIFY.UseVisualStyleBackColor = True
'
'CH_COPY_CHANNEL_USER_IMAGE
'
Me.CH_COPY_CHANNEL_USER_IMAGE.AutoSize = True
@@ -593,7 +605,7 @@
TAB_BASIS.Location = New System.Drawing.Point(4, 22)
TAB_BASIS.Name = "TAB_BASIS"
TAB_BASIS.Padding = New System.Windows.Forms.Padding(3)
TAB_BASIS.Size = New System.Drawing.Size(576, 347)
TAB_BASIS.Size = New System.Drawing.Size(576, 372)
TAB_BASIS.TabIndex = 0
TAB_BASIS.Text = "Basis"
'
@@ -603,7 +615,7 @@
TAB_DEFAULTS.Location = New System.Drawing.Point(4, 22)
TAB_DEFAULTS.Name = "TAB_DEFAULTS"
TAB_DEFAULTS.Padding = New System.Windows.Forms.Padding(3)
TAB_DEFAULTS.Size = New System.Drawing.Size(576, 358)
TAB_DEFAULTS.Size = New System.Drawing.Size(576, 372)
TAB_DEFAULTS.TabIndex = 1
TAB_DEFAULTS.Text = "Defaults"
'
@@ -625,7 +637,7 @@
TP_DEFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
TP_DEFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
TP_DEFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_DEFS.Size = New System.Drawing.Size(570, 352)
TP_DEFS.Size = New System.Drawing.Size(570, 366)
TP_DEFS.TabIndex = 0
'
'TAB_DEFS_CHANNELS
@@ -634,7 +646,7 @@
TAB_DEFS_CHANNELS.Location = New System.Drawing.Point(4, 22)
TAB_DEFS_CHANNELS.Name = "TAB_DEFS_CHANNELS"
TAB_DEFS_CHANNELS.Padding = New System.Windows.Forms.Padding(3)
TAB_DEFS_CHANNELS.Size = New System.Drawing.Size(576, 358)
TAB_DEFS_CHANNELS.Size = New System.Drawing.Size(576, 372)
TAB_DEFS_CHANNELS.TabIndex = 4
TAB_DEFS_CHANNELS.Text = "Channels"
'
@@ -658,7 +670,7 @@
TP_CHANNELS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_CHANNELS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 20.0!))
TP_CHANNELS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 20.0!))
TP_CHANNELS.Size = New System.Drawing.Size(570, 352)
TP_CHANNELS.Size = New System.Drawing.Size(570, 366)
TP_CHANNELS.TabIndex = 0
'
'TXT_CHANNEL_USER_POST_LIMIT
@@ -687,7 +699,7 @@
TAB_DEFS_REDDIT.Location = New System.Drawing.Point(4, 22)
TAB_DEFS_REDDIT.Name = "TAB_DEFS_REDDIT"
TAB_DEFS_REDDIT.Padding = New System.Windows.Forms.Padding(3)
TAB_DEFS_REDDIT.Size = New System.Drawing.Size(576, 358)
TAB_DEFS_REDDIT.Size = New System.Drawing.Size(576, 372)
TAB_DEFS_REDDIT.TabIndex = 2
TAB_DEFS_REDDIT.Text = "Reddit"
'
@@ -696,27 +708,29 @@
Me.DEFS_REDDIT.CellBorderStyle = System.Windows.Forms.TableLayoutPanelCellBorderStyle.[Single]
Me.DEFS_REDDIT.ColumnCount = 1
Me.DEFS_REDDIT.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.DEFS_REDDIT.Controls.Add(Me.TXT_REDDIT_SAVED_POSTS_USER, 0, 3)
Me.DEFS_REDDIT.Controls.Add(Me.TXT_REDDIT_SAVED_POSTS_USER, 0, 4)
Me.DEFS_REDDIT.Controls.Add(Me.CH_REDDIT_USER_MEDIA, 0, 3)
Me.DEFS_REDDIT.Dock = System.Windows.Forms.DockStyle.Fill
Me.DEFS_REDDIT.Location = New System.Drawing.Point(3, 3)
Me.DEFS_REDDIT.Name = "DEFS_REDDIT"
Me.DEFS_REDDIT.RowCount = 5
Me.DEFS_REDDIT.RowCount = 6
Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.DEFS_REDDIT.Size = New System.Drawing.Size(570, 352)
Me.DEFS_REDDIT.Size = New System.Drawing.Size(570, 366)
Me.DEFS_REDDIT.TabIndex = 1
'
'TXT_REDDIT_SAVED_POSTS_USER
'
Me.TXT_REDDIT_SAVED_POSTS_USER.CaptionText = "Saved posts user"
Me.TXT_REDDIT_SAVED_POSTS_USER.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_REDDIT_SAVED_POSTS_USER.Location = New System.Drawing.Point(4, 82)
Me.TXT_REDDIT_SAVED_POSTS_USER.Location = New System.Drawing.Point(4, 108)
Me.TXT_REDDIT_SAVED_POSTS_USER.Name = "TXT_REDDIT_SAVED_POSTS_USER"
Me.TXT_REDDIT_SAVED_POSTS_USER.Size = New System.Drawing.Size(562, 22)
Me.TXT_REDDIT_SAVED_POSTS_USER.TabIndex = 3
Me.TXT_REDDIT_SAVED_POSTS_USER.TabIndex = 4
'
'TAB_DEFS_TWITTER
'
@@ -724,7 +738,7 @@
TAB_DEFS_TWITTER.Location = New System.Drawing.Point(4, 22)
TAB_DEFS_TWITTER.Name = "TAB_DEFS_TWITTER"
TAB_DEFS_TWITTER.Padding = New System.Windows.Forms.Padding(3)
TAB_DEFS_TWITTER.Size = New System.Drawing.Size(576, 358)
TAB_DEFS_TWITTER.Size = New System.Drawing.Size(576, 372)
TAB_DEFS_TWITTER.TabIndex = 3
TAB_DEFS_TWITTER.Text = "Twitter"
'
@@ -743,7 +757,7 @@
Me.DEFS_TWITTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_TWITTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_TWITTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.DEFS_TWITTER.Size = New System.Drawing.Size(570, 352)
Me.DEFS_TWITTER.Size = New System.Drawing.Size(570, 366)
Me.DEFS_TWITTER.TabIndex = 1
'
'CH_TWITTER_USER_MEDIA
@@ -800,7 +814,7 @@
Me.TAB_MAIN.Location = New System.Drawing.Point(0, 0)
Me.TAB_MAIN.Name = "TAB_MAIN"
Me.TAB_MAIN.SelectedIndex = 0
Me.TAB_MAIN.Size = New System.Drawing.Size(584, 373)
Me.TAB_MAIN.Size = New System.Drawing.Size(584, 398)
Me.TAB_MAIN.TabIndex = 1
'
'TAB_DEFS_INSTAGRAM
@@ -809,7 +823,7 @@
Me.TAB_DEFS_INSTAGRAM.Controls.Add(Me.DEFS_INST)
Me.TAB_DEFS_INSTAGRAM.Location = New System.Drawing.Point(4, 22)
Me.TAB_DEFS_INSTAGRAM.Name = "TAB_DEFS_INSTAGRAM"
Me.TAB_DEFS_INSTAGRAM.Size = New System.Drawing.Size(576, 358)
Me.TAB_DEFS_INSTAGRAM.Size = New System.Drawing.Size(576, 372)
Me.TAB_DEFS_INSTAGRAM.TabIndex = 5
Me.TAB_DEFS_INSTAGRAM.Text = "Instagram"
'
@@ -835,7 +849,7 @@
Me.DEFS_INST.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEFS_INST.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEFS_INST.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.DEFS_INST.Size = New System.Drawing.Size(576, 358)
Me.DEFS_INST.Size = New System.Drawing.Size(576, 372)
Me.DEFS_INST.TabIndex = 1
'
'TXT_INST_SAVED_POSTS_USER
@@ -854,7 +868,7 @@
Me.TAB_DEFS_REDGIFS.Controls.Add(Me.DEFS_REDGIFS)
Me.TAB_DEFS_REDGIFS.Location = New System.Drawing.Point(4, 22)
Me.TAB_DEFS_REDGIFS.Name = "TAB_DEFS_REDGIFS"
Me.TAB_DEFS_REDGIFS.Size = New System.Drawing.Size(576, 358)
Me.TAB_DEFS_REDGIFS.Size = New System.Drawing.Size(576, 372)
Me.TAB_DEFS_REDGIFS.TabIndex = 6
Me.TAB_DEFS_REDGIFS.Text = "RedGifs"
'
@@ -871,7 +885,7 @@
Me.DEFS_REDGIFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDGIFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDGIFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.DEFS_REDGIFS.Size = New System.Drawing.Size(576, 358)
Me.DEFS_REDGIFS.Size = New System.Drawing.Size(576, 372)
Me.DEFS_REDGIFS.TabIndex = 0
'
'CONTAINER_MAIN
@@ -880,7 +894,7 @@
'CONTAINER_MAIN.ContentPanel
'
Me.CONTAINER_MAIN.ContentPanel.Controls.Add(Me.TAB_MAIN)
Me.CONTAINER_MAIN.ContentPanel.Size = New System.Drawing.Size(584, 373)
Me.CONTAINER_MAIN.ContentPanel.Size = New System.Drawing.Size(584, 398)
Me.CONTAINER_MAIN.Dock = System.Windows.Forms.DockStyle.Fill
Me.CONTAINER_MAIN.LeftToolStripPanelVisible = False
Me.CONTAINER_MAIN.Location = New System.Drawing.Point(0, 0)
@@ -890,16 +904,16 @@
Me.CONTAINER_MAIN.TabIndex = 0
Me.CONTAINER_MAIN.TopToolStripPanelVisible = False
'
'CH_SHOW_NOTIFY
'CH_REDDIT_USER_MEDIA
'
Me.CH_SHOW_NOTIFY.AutoSize = True
Me.CH_SHOW_NOTIFY.Dock = System.Windows.Forms.DockStyle.Fill
Me.CH_SHOW_NOTIFY.Location = New System.Drawing.Point(4, 318)
Me.CH_SHOW_NOTIFY.Name = "CH_SHOW_NOTIFY"
Me.CH_SHOW_NOTIFY.Size = New System.Drawing.Size(562, 19)
Me.CH_SHOW_NOTIFY.TabIndex = 11
Me.CH_SHOW_NOTIFY.Text = "Show notifications"
Me.CH_SHOW_NOTIFY.UseVisualStyleBackColor = True
Me.CH_REDDIT_USER_MEDIA.AutoSize = True
Me.CH_REDDIT_USER_MEDIA.Dock = System.Windows.Forms.DockStyle.Fill
Me.CH_REDDIT_USER_MEDIA.Location = New System.Drawing.Point(4, 82)
Me.CH_REDDIT_USER_MEDIA.Name = "CH_REDDIT_USER_MEDIA"
Me.CH_REDDIT_USER_MEDIA.Size = New System.Drawing.Size(562, 19)
Me.CH_REDDIT_USER_MEDIA.TabIndex = 3
Me.CH_REDDIT_USER_MEDIA.Text = "Get user media only"
Me.CH_REDDIT_USER_MEDIA.UseVisualStyleBackColor = True
'
'GlobalSettingsForm
'
@@ -945,6 +959,7 @@
CType(Me.TXT_CHANNEL_USER_POST_LIMIT, System.ComponentModel.ISupportInitialize).EndInit()
TAB_DEFS_REDDIT.ResumeLayout(False)
Me.DEFS_REDDIT.ResumeLayout(False)
Me.DEFS_REDDIT.PerformLayout()
CType(Me.TXT_REDDIT_SAVED_POSTS_USER, System.ComponentModel.ISupportInitialize).EndInit()
TAB_DEFS_TWITTER.ResumeLayout(False)
Me.DEFS_TWITTER.ResumeLayout(False)
@@ -998,12 +1013,13 @@
Private WithEvents TXT_REQ_COUNT As PersonalUtilities.Forms.Controls.TextBoxExtended
Private WithEvents TXT_LIMIT_TIMER As PersonalUtilities.Forms.Controls.TextBoxExtended
Private WithEvents TAB_DEFS_REDGIFS As TabPage
Friend WithEvents TAB_MAIN As TabControl
Private WithEvents TAB_MAIN As TabControl
Private WithEvents DEFS_TWITTER As SiteDefaults
Private WithEvents DEFS_REDDIT As SiteDefaults
Private WithEvents DEFS_INST As SiteDefaults
Private WithEvents DEFS_REDGIFS As SiteDefaults
Private WithEvents TXT_INST_SAVED_POSTS_USER As PersonalUtilities.Forms.Controls.TextBoxExtended
Private WithEvents CH_SHOW_NOTIFY As CheckBox
Private WithEvents CH_REDDIT_USER_MEDIA As CheckBox
End Class
End Namespace

2
SCrawler/Editors/GlobalSettingsForm.vb
View File

@@ -91,6 +91,7 @@ Namespace Editors
'Reddit
With .Site(Sites.Reddit)
SetChecker(DEFS_REDDIT, Sites.Reddit)
CH_REDDIT_USER_MEDIA.Checked = .GetUserMediaOnly
TXT_REDDIT_SAVED_POSTS_USER.Text = .SavedPostsUserName
End With
'Twitter
@@ -226,6 +227,7 @@ Namespace Editors
'Reddit
With .Site(Sites.Reddit)
SetPropByChecker(DEFS_REDDIT, Sites.Reddit)
.GetUserMediaOnly.Value = CH_REDDIT_USER_MEDIA.Checked
.SavedPostsUserName.Value = TXT_REDDIT_SAVED_POSTS_USER.Text
End With
'Twitter

17
SCrawler/Editors/UserCreatorForm.vb
View File

@@ -33,11 +33,7 @@ Namespace Editors
End Property
Friend ReadOnly Property UserMediaOnly As Boolean
Get
If User.Site = Sites.Twitter Then
Return CH_PARSE_USER_MEDIA.Checked
Else
Return False
End If
Return CH_PARSE_USER_MEDIA.Checked
End Get
End Property
Friend ReadOnly Property UserReady As Boolean
@@ -105,7 +101,6 @@ Namespace Editors
OPT_REDDIT.Checked = False
OPT_TWITTER.Checked = False
OPT_INSTAGRAM.Checked = False
CH_PARSE_USER_MEDIA.Enabled = False
CH_READY_FOR_DOWN.Checked = True
CH_TEMP.Checked = Settings.DefaultTemporary
CH_DOWN_IMAGES.Checked = Settings.DefaultDownloadImages
@@ -115,7 +110,7 @@ Namespace Editors
TXT_USER.Text = User.Name
TXT_SPEC_FOLDER.Text = User.SpecialPath
Select Case User.Site
Case Sites.Reddit : OPT_REDDIT.Checked = True : CH_PARSE_USER_MEDIA.Enabled = False
Case Sites.Reddit : OPT_REDDIT.Checked = True
Case Sites.Twitter : OPT_TWITTER.Checked = True
Case Sites.Instagram : OPT_INSTAGRAM.Checked = True
Case Sites.RedGifs : OPT_REDGIFS.Checked = True
@@ -212,7 +207,7 @@ Namespace Editors
Else
.Labels.ListAddList(UserLabels, LAP.NotContainsOnly, LAP.ClearBeforeAdd)
End If
If OPT_TWITTER.Checked Then .ParseUserMediaOnly = CH_PARSE_USER_MEDIA.Checked
.ParseUserMediaOnly = CH_PARSE_USER_MEDIA.Checked
.UpdateUserInformation()
End With
End If
@@ -312,8 +307,7 @@ CloseForm:
CH_TEMP.Checked = .Temporary
CH_DOWN_IMAGES.Checked = .DownloadImages
CH_DOWN_VIDEOS.Checked = .DownloadVideos
CH_PARSE_USER_MEDIA.Checked = s = Sites.Twitter AndAlso .GetUserMediaOnly.Value
CH_PARSE_USER_MEDIA.Enabled = s = Sites.Twitter
CH_PARSE_USER_MEDIA.Checked = .GetUserMediaOnly.Value
CH_READY_FOR_DOWN.Checked = Not CH_TEMP.Checked
End With
End If
@@ -322,7 +316,6 @@ CloseForm:
If CH_ADD_BY_LIST.Checked Then
TXT_DESCR.GroupBoxText = "Users list"
CH_AUTO_DETECT_SITE.Enabled = True
CH_PARSE_USER_MEDIA.Enabled = True
Else
TXT_DESCR.GroupBoxText = "Description"
CH_AUTO_DETECT_SITE.Checked = False
@@ -396,7 +389,7 @@ CloseForm:
.DownloadImages = CH_DOWN_IMAGES.Checked
.DownloadVideos = CH_DOWN_VIDEOS.Checked
.Labels.ListAddList(UserLabels)
If s = Sites.Twitter Then .ParseUserMediaOnly = CH_PARSE_USER_MEDIA.Checked
.ParseUserMediaOnly = CH_PARSE_USER_MEDIA.Checked
.UpdateUserInformation()
End With
Added += 1

3
SCrawler/MainFrame.vb
View File

@@ -330,7 +330,8 @@ CloseResume:
If f.StartIndex >= 0 Then
OnUsersAddedHandler(f.StartIndex)
Else
i = Settings.Users.FindIndex(Function(u) u.Site = f.User.Site And u.Name = f.User.Name)
Dim SimpleUser As Predicate(Of IUserData) = Function(u) u.Site = f.User.Site And u.Name = f.User.Name
i = Settings.Users.FindIndex(Function(u) If(u.IsCollection, DirectCast(u, UserDataBind).Collections.Exists(SimpleUser), SimpleUser.Invoke(u)))
If i < 0 Then
If Not UserBanned(f.User.Name) Then
Settings.UpdateUsersList(f.User)

4
SCrawler/My Project/AssemblyInfo.vb
View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("2.0.0.2")>
<Assembly: AssemblyFileVersion("2.0.0.2")>
<Assembly: AssemblyVersion("2.0.0.3")>
<Assembly: AssemblyFileVersion("2.0.0.3")>
<Assembly: NeutralResourcesLanguage("en")>

15
SCrawler/SettingsCLS.vb
View File

@@ -51,11 +51,8 @@ Friend Class SettingsCLS : Implements IDisposable
Dim n() As String = {"Defaults"}
DefaultTemporary = New XMLValue(Of Boolean)("Temporary", False, MyXML, n)
DefaultTemporary.ReplaceByValue("DefaultTemporary")
DefaultDownloadImages = New XMLValue(Of Boolean)("DownloadImages", True, MyXML, n)
DefaultDownloadImages.ReplaceByValue("DefaultDownloadImages")
DefaultDownloadVideos = New XMLValue(Of Boolean)("DownloadVideos", True, MyXML, n)
DefaultDownloadVideos.ReplaceByValue("DefaultDownloadVideos")
ChangeReadyForDownOnTempChange = New XMLValue(Of Boolean)("ChangeReadyForDownOnTempChange", True, MyXML, n)
MySites = New Dictionary(Of Sites, SiteSettings) From {
@@ -89,33 +86,21 @@ Friend Class SettingsCLS : Implements IDisposable
ChannelsDefaultTemporary = New XMLValue(Of Boolean)("ChannelsDefaultTemporary", True, MyXML, n)
ChannelsRegularCheckMD5 = New XMLValue(Of Boolean)("ChannelsRegularCheckMD5", False, MyXML, n)
ChannelsImagesRows = New XMLValue(Of Integer)("ImagesRows", 2, MyXML, n)
ChannelsImagesRows.ReplaceByValue("ChannelsImagesRows")
ChannelsImagesColumns = New XMLValue(Of Integer)("ImagesColumns", 5, MyXML, n)
ChannelsImagesColumns.ReplaceByValue("ChannelsImagesColumns")
ChannelsHideExistsUser = New XMLValue(Of Boolean)("HideExistsUser", True, MyXML, n)
ChannelsHideExistsUser.ReplaceByValue("ChannelsHideExistsUser")
ChannelsMaxJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks, MyXML, n)
ChannelsMaxJobsCount.ReplaceByValue("ChannelsMaxJobsCount")
n = {"Users"}
FromChannelDownloadTop = New XMLValue(Of Integer)("FromChannelDownloadTop", 10, MyXML, n)
FromChannelDownloadTop.ReplaceByValue("FromChannelDownloadTop")
FromChannelDownloadTopUse = New XMLValue(Of Boolean)("FromChannelDownloadTopUse", False, MyXML, n)
FromChannelDownloadTopUse.ReplaceByValue("FromChannelDownloadTopUse")
FromChannelCopyImageToUser = New XMLValue(Of Boolean)("FromChannelCopyImageToUser", True, MyXML, n)
FromChannelCopyImageToUser.ReplaceByValue("FromChannelCopyImageToUser")
n = {"Users", "FileName"}
MaxUsersJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks, MyXML, n)
MaxUsersJobsCount.ReplaceByValue("MaxUsersJobsCount")
FileAddDateToFileName = New XMLValue(Of Boolean)("FileAddDateToFileName", False, MyXML, n) With {.OnChangeFunction = AddressOf ChangeDateProvider}
FileAddDateToFileName.ReplaceByValue("FileAddDateToFileName")
FileAddTimeToFileName = New XMLValue(Of Boolean)("FileAddTimeToFileName", False, MyXML, n) With {.OnChangeFunction = AddressOf ChangeDateProvider}
FileAddTimeToFileName.ReplaceByValue("FileAddTimeToFileName")
FileDateTimePositionEnd = New XMLValue(Of Boolean)("FileDateTimePositionEnd", True, MyXML, n) With {.OnChangeFunction = AddressOf ChangeDateProvider}
FileDateTimePositionEnd.ReplaceByValue("FileDateTimePositionEnd")
FileReplaceNameByDate = New XMLValue(Of Boolean)("FileReplaceNameByDate", False, MyXML, n)
FileReplaceNameByDate.ReplaceByValue("FileReplaceNameByDate")
CheckUpdatesAtStart = New XMLValue(Of Boolean)("CheckUpdatesAtStart", True, MyXML)
ShowNewVersionNotification = New XMLValue(Of Boolean)("ShowNewVersionNotification", True, MyXML)

4
SCrawler/TDownloader.vb
View File

@@ -184,8 +184,8 @@ Friend Class TDownloader : Implements IDisposable
If _Job.Count > 0 Then
Const nf As ANumbers.Formats = ANumbers.Formats.Number
Dim t As New List(Of Task)
Dim i% = -1
Dim j% = Settings.MaxUsersJobsCount - 1
Dim i% = 0
Dim j% = Settings.MaxUsersJobsCount
Dim limit% = IIf(_Job.Site = Sites.Instagram, 1, j)
Dim Keys As New List(Of String)
Dim h As Boolean = False