From 5d5ce29f3696d9d17a545ab7524a57b81fc5f12f Mon Sep 17 00:00:00 2001
From: Andy <88590076+AAndyProgram@users.noreply.github.com>
Date: Wed, 2 Feb 2022 07:34:36 +0300
Subject: [PATCH] 2.0.0.3
Added GetUserMediaOnly for Reddit
Fixed Reddit CrossPosts issue
Fixed validating new users (collections)
Fixed incorrect count of Instagram download tasks
---
Changelog.md | 15 ++++
SCrawler/API/Base/SiteSettings.vb | 8 +-
SCrawler/API/Reddit/UserData.vb | 59 +++++++++-----
.../Editors/GlobalSettingsForm.Designer.vb | 78 +++++++++++--------
SCrawler/Editors/GlobalSettingsForm.vb | 2 +
SCrawler/Editors/UserCreatorForm.vb | 17 ++--
SCrawler/MainFrame.vb | 3 +-
SCrawler/My Project/AssemblyInfo.vb | 4 +-
SCrawler/SettingsCLS.vb | 15 ----
SCrawler/TDownloader.vb | 4 +-
10 files changed, 115 insertions(+), 90 deletions(-)
diff --git a/Changelog.md b/Changelog.md
index 9594669..d607d3b 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -1,3 +1,18 @@
+# 2.0.0.3
+
+**Removed compatibility of program settings with version 1.0.0.4 and lower.**
+
+**If your program version is 1.0.0.4 and lower, it is strongly recommended that you upgrade to release 2.0.0.2 to update the program settings (and run the program). Then update to this release. Otherwise, you will have to configure the program settings again**
+
+**If your program version is 1.0.1.0 or higher, you should not pay attention to this message.**
+
+- Added
+ - The "Get User Media Only" setting is now available for Reddit. If checked then "CrossPosts" will be skipped, otherwise "CrossPosts" will be included.
+- Fixed
+ - In some cases, the program did not parse all Reddit posts.
+ - Collection ignored when validated when creating a new user
+ - Incorrect number of Instagram profiles downloads per session
+
# 2.0.0.2
**This is the last release that supports program settings of version 1.0.0.4 and lower. Compatibility of program settings with version 1.0.0.4 and lower will be removed in future releases. It is strongly recommended that you upgrade to this release before future releases. Otherwise, you will have to configure the program settings again. If your program version is 1.0.1.0 or higher, you should not pay attention to this message.**
diff --git a/SCrawler/API/Base/SiteSettings.vb b/SCrawler/API/Base/SiteSettings.vb
index 2652bb9..d4dace3 100644
--- a/SCrawler/API/Base/SiteSettings.vb
+++ b/SCrawler/API/Base/SiteSettings.vb
@@ -123,7 +123,6 @@ Namespace API.Base
Dim n() As String = {SettingsCLS.Name_Node_Sites, Site.ToString}
_Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"), _XML, n, XMLValue(Of SFile).ToFilePath)
- _Path.ReplaceByValue("Path", {Site.ToString})
_XML.Remove(Site.ToString)
Temporary = New XMLValue(Of Boolean)
@@ -138,12 +137,7 @@ Namespace API.Base
DownloadVideos.SetExtended("DownloadVideos", True, _XML, n)
DownloadVideos.SetDefault(_Vids)
- If Site = Sites.Twitter Then
- GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n)
- GetUserMediaOnly.ReplaceByValue("TwitterDefaultGetUserMedia", n)
- Else
- GetUserMediaOnly = New XMLValue(Of Boolean)
- End If
+ GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n)
CreateProp(InstaHashUpdateRequired, Sites.Instagram, "InstaHashUpdateRequired", True, _XML, n)
CreateProp(InstaHash, Sites.Instagram, "InstaHash", String.Empty, _XML, n)
diff --git a/SCrawler/API/Reddit/UserData.vb b/SCrawler/API/Reddit/UserData.vb
index 83747aa..ac12b22 100644
--- a/SCrawler/API/Reddit/UserData.vb
+++ b/SCrawler/API/Reddit/UserData.vb
@@ -54,12 +54,14 @@ Namespace API.Reddit
Private Sub New()
ChannelPostsNames = New List(Of String)
_ExistsUsersNames = New List(Of String)
+ _CrossPosts = New List(Of String)
End Sub
''' Default initializer
Friend Sub New(ByVal u As UserInfo, Optional ByVal _LoadUserInformation As Boolean = True, Optional ByVal InvokeImageHandler As Boolean = True)
MyBase.New(InvokeImageHandler)
ChannelPostsNames = New List(Of String)
_ExistsUsersNames = New List(Of String)
+ _CrossPosts = New List(Of String)
User = u
If _LoadUserInformation Then LoadUserInformation()
End Sub
@@ -108,10 +110,13 @@ Namespace API.Reddit
#End Region
#Region "Download Functions (User, Channel)"
Private _TotalPostsDownloaded As Integer = 0
+ Private ReadOnly _CrossPosts As List(Of String)
Private Sub DownloadDataUser(ByVal POST As String, ByVal Token As CancellationToken)
+ Const CPRI$ = "crosspostRootId"
+ Const CPPI$ = "crosspostParentId"
Dim URL$ = String.Empty
Try
- Dim PostID$ = String.Empty
+ Dim PostID$ = String.Empty, PostTmp$ = String.Empty
Dim PostDate$
Dim n As EContainer, nn As EContainer, s As EContainer
Dim NewPostDetected As Boolean = False
@@ -120,8 +125,10 @@ Namespace API.Reddit
Dim added As Boolean
Dim __ItemType$
Dim tmpType As UTypes
- Dim CheckNode As Predicate(Of EContainer) = Function(e) e("author").XmlIfNothingValue("/").ToLower.Equals(Name.ToLower)
+ Dim IsCrossPost As Predicate(Of EContainer) = Function(e) Not (e.Value(CPRI).IsEmptyString And e.Value(CPPI).IsEmptyString)
+ Dim CheckNode As Predicate(Of EContainer) = Function(e) Not ParseUserMediaOnly OrElse e("author").XmlIfNothingValue("/").ToLower.Equals(Name.ToLower)
Dim UPicType As Func(Of String, UTypes) = Function(input) IIf(input = "image", UTypes.Picture, UTypes.GIF)
+ Dim _PostID As Func(Of String) = Function() IIf(PostTmp.IsEmptyString, PostID, PostTmp)
URL = $"https://gateway.reddit.com/desktopapi/v1/user/{Name}/posts?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort=new&t=all&layout=classic"
ThrowAny(Token)
@@ -134,39 +141,51 @@ Namespace API.Reddit
For Each nn In n
ThrowAny(Token)
If nn.Count > 0 Then
- PostID = nn.Name
- If PostID.IsEmptyString AndAlso nn.Contains("id") Then PostID = nn("id").Value
- If nn.Contains("created") Then PostDate = nn("created").Value Else PostDate = String.Empty
- If Not _TempPostsList.Contains(PostID) Then
- NewPostDetected = True
- _TempPostsList.Add(PostID)
- Else
- ExistsDetected = True
- Continue For
- End If
-
If CheckNode(nn) Then
+
+ 'Obtain post ID
+ PostTmp = nn.Name
+ If PostTmp.IsEmptyString Then PostTmp = nn.Value("id")
+ If PostTmp.IsEmptyString Then Continue For
+ 'Check for CrossPost
+ If IsCrossPost(nn) Then
+ _CrossPosts.ListAddList({nn.Value(CPRI), nn.Value(CPPI)}, LNC)
+ Continue For
+ Else
+ If Not _CrossPosts.Contains(PostTmp) Then PostID = PostTmp : PostTmp = String.Empty
+ End If
+
+ 'Download decision
+ If Not _TempPostsList.Contains(_PostID()) Then
+ NewPostDetected = True
+ _TempPostsList.Add(_PostID())
+ Else
+ If Not _CrossPosts.Contains(_PostID()) Then ExistsDetected = True
+ Continue For
+ End If
+ If nn.Contains("created") Then PostDate = nn("created").Value Else PostDate = String.Empty
+
_ItemsBefore = _TempMediaList.Count
added = True
s = nn.ItemF({"source", "url"})
If s.XmlIfNothingValue("/").Contains("redgifs.com") Then
- _TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, PostID, PostDate,, IsChannel), LNC)
- ElseIf Not CreateImgurMedia(s.XmlIfNothingValue, PostID, PostDate,, IsChannel) Then
+ _TempMediaList.ListAddValue(MediaFromData(UTypes.VideoPre, s.Value, _PostID(), PostDate,, IsChannel), LNC)
+ ElseIf Not CreateImgurMedia(s.XmlIfNothingValue, _PostID(), PostDate,, IsChannel) Then
s = nn.ItemF({"media"}).XmlIfNothing
__ItemType = s("type").XmlIfNothingValue
Select Case __ItemType
- Case "gallery" : If Not DownloadGallery(s, PostID, PostDate) Then added = False
+ Case "gallery" : If Not DownloadGallery(s, _PostID(), PostDate) Then added = False
Case "image", "gifvideo"
If s.Contains("content") Then
_TempMediaList.ListAddValue(MediaFromData(UPicType(__ItemType), s.Value("content"),
- PostID, PostDate,, IsChannel), LNC)
+ _PostID(), PostDate,, IsChannel), LNC)
Else
added = False
End If
Case "video"
If Settings.UseM3U8 AndAlso s("hlsUrl").XmlIfNothingValue("/").ToLower.Contains("m3u8") Then
_TempMediaList.ListAddValue(MediaFromData(UTypes.m3u8, s.Value("hlsUrl"),
- PostID, PostDate,, IsChannel), LNC)
+ _PostID(), PostDate,, IsChannel), LNC)
Else
added = False
End If
@@ -186,7 +205,7 @@ Namespace API.Reddit
End Select
End With
If Not tmpType = UTypes.Undefined Then
- _TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, PostID, PostDate,, IsChannel), LNC)
+ _TempMediaList.ListAddValue(MediaFromData(tmpType, s.Value, _PostID(), PostDate,, IsChannel), LNC)
End If
End If
End If
@@ -596,7 +615,7 @@ Namespace API.Reddit
Return 1
End Function
Protected Overrides Sub Dispose(ByVal disposing As Boolean)
- If Not disposedValue And disposing Then ChannelPostsNames.Clear() : _ExistsUsersNames.Clear()
+ If Not disposedValue And disposing Then ChannelPostsNames.Clear() : _ExistsUsersNames.Clear() : _CrossPosts.Clear()
MyBase.Dispose(disposing)
End Sub
End Class
diff --git a/SCrawler/Editors/GlobalSettingsForm.Designer.vb b/SCrawler/Editors/GlobalSettingsForm.Designer.vb
index e84d4d3..883d83a 100644
--- a/SCrawler/Editors/GlobalSettingsForm.Designer.vb
+++ b/SCrawler/Editors/GlobalSettingsForm.Designer.vb
@@ -53,6 +53,7 @@
Me.OPT_FILE_DATE_END = New System.Windows.Forms.RadioButton()
Me.CH_EXIT_CONFIRM = New System.Windows.Forms.CheckBox()
Me.CH_CLOSE_TO_TRAY = New System.Windows.Forms.CheckBox()
+ Me.CH_SHOW_NOTIFY = New System.Windows.Forms.CheckBox()
Me.CH_COPY_CHANNEL_USER_IMAGE = New System.Windows.Forms.CheckBox()
Me.CH_DEF_TEMP = New System.Windows.Forms.CheckBox()
Me.CH_DOWN_IMAGES = New System.Windows.Forms.CheckBox()
@@ -76,7 +77,7 @@
Me.TAB_DEFS_REDGIFS = New System.Windows.Forms.TabPage()
Me.DEFS_REDGIFS = New SCrawler.Editors.SiteDefaults()
Me.CONTAINER_MAIN = New System.Windows.Forms.ToolStripContainer()
- Me.CH_SHOW_NOTIFY = New System.Windows.Forms.CheckBox()
+ Me.CH_REDDIT_USER_MEDIA = New System.Windows.Forms.CheckBox()
TP_BASIS = New System.Windows.Forms.TableLayoutPanel()
TP_IMAGES = New System.Windows.Forms.TableLayoutPanel()
TP_FILE_NAME = New System.Windows.Forms.TableLayoutPanel()
@@ -162,7 +163,7 @@
TP_BASIS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
TP_BASIS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
TP_BASIS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
- TP_BASIS.Size = New System.Drawing.Size(570, 341)
+ TP_BASIS.Size = New System.Drawing.Size(570, 366)
TP_BASIS.TabIndex = 0
'
'TXT_GLOBAL_PATH
@@ -469,6 +470,17 @@
Me.CH_CLOSE_TO_TRAY.Text = "Close to tray"
Me.CH_CLOSE_TO_TRAY.UseVisualStyleBackColor = True
'
+ 'CH_SHOW_NOTIFY
+ '
+ Me.CH_SHOW_NOTIFY.AutoSize = True
+ Me.CH_SHOW_NOTIFY.Dock = System.Windows.Forms.DockStyle.Fill
+ Me.CH_SHOW_NOTIFY.Location = New System.Drawing.Point(4, 318)
+ Me.CH_SHOW_NOTIFY.Name = "CH_SHOW_NOTIFY"
+ Me.CH_SHOW_NOTIFY.Size = New System.Drawing.Size(562, 19)
+ Me.CH_SHOW_NOTIFY.TabIndex = 11
+ Me.CH_SHOW_NOTIFY.Text = "Show notifications"
+ Me.CH_SHOW_NOTIFY.UseVisualStyleBackColor = True
+ '
'CH_COPY_CHANNEL_USER_IMAGE
'
Me.CH_COPY_CHANNEL_USER_IMAGE.AutoSize = True
@@ -593,7 +605,7 @@
TAB_BASIS.Location = New System.Drawing.Point(4, 22)
TAB_BASIS.Name = "TAB_BASIS"
TAB_BASIS.Padding = New System.Windows.Forms.Padding(3)
- TAB_BASIS.Size = New System.Drawing.Size(576, 347)
+ TAB_BASIS.Size = New System.Drawing.Size(576, 372)
TAB_BASIS.TabIndex = 0
TAB_BASIS.Text = "Basis"
'
@@ -603,7 +615,7 @@
TAB_DEFAULTS.Location = New System.Drawing.Point(4, 22)
TAB_DEFAULTS.Name = "TAB_DEFAULTS"
TAB_DEFAULTS.Padding = New System.Windows.Forms.Padding(3)
- TAB_DEFAULTS.Size = New System.Drawing.Size(576, 358)
+ TAB_DEFAULTS.Size = New System.Drawing.Size(576, 372)
TAB_DEFAULTS.TabIndex = 1
TAB_DEFAULTS.Text = "Defaults"
'
@@ -625,7 +637,7 @@
TP_DEFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
TP_DEFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
TP_DEFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
- TP_DEFS.Size = New System.Drawing.Size(570, 352)
+ TP_DEFS.Size = New System.Drawing.Size(570, 366)
TP_DEFS.TabIndex = 0
'
'TAB_DEFS_CHANNELS
@@ -634,7 +646,7 @@
TAB_DEFS_CHANNELS.Location = New System.Drawing.Point(4, 22)
TAB_DEFS_CHANNELS.Name = "TAB_DEFS_CHANNELS"
TAB_DEFS_CHANNELS.Padding = New System.Windows.Forms.Padding(3)
- TAB_DEFS_CHANNELS.Size = New System.Drawing.Size(576, 358)
+ TAB_DEFS_CHANNELS.Size = New System.Drawing.Size(576, 372)
TAB_DEFS_CHANNELS.TabIndex = 4
TAB_DEFS_CHANNELS.Text = "Channels"
'
@@ -658,7 +670,7 @@
TP_CHANNELS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_CHANNELS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 20.0!))
TP_CHANNELS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 20.0!))
- TP_CHANNELS.Size = New System.Drawing.Size(570, 352)
+ TP_CHANNELS.Size = New System.Drawing.Size(570, 366)
TP_CHANNELS.TabIndex = 0
'
'TXT_CHANNEL_USER_POST_LIMIT
@@ -687,7 +699,7 @@
TAB_DEFS_REDDIT.Location = New System.Drawing.Point(4, 22)
TAB_DEFS_REDDIT.Name = "TAB_DEFS_REDDIT"
TAB_DEFS_REDDIT.Padding = New System.Windows.Forms.Padding(3)
- TAB_DEFS_REDDIT.Size = New System.Drawing.Size(576, 358)
+ TAB_DEFS_REDDIT.Size = New System.Drawing.Size(576, 372)
TAB_DEFS_REDDIT.TabIndex = 2
TAB_DEFS_REDDIT.Text = "Reddit"
'
@@ -696,27 +708,29 @@
Me.DEFS_REDDIT.CellBorderStyle = System.Windows.Forms.TableLayoutPanelCellBorderStyle.[Single]
Me.DEFS_REDDIT.ColumnCount = 1
Me.DEFS_REDDIT.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
- Me.DEFS_REDDIT.Controls.Add(Me.TXT_REDDIT_SAVED_POSTS_USER, 0, 3)
+ Me.DEFS_REDDIT.Controls.Add(Me.TXT_REDDIT_SAVED_POSTS_USER, 0, 4)
+ Me.DEFS_REDDIT.Controls.Add(Me.CH_REDDIT_USER_MEDIA, 0, 3)
Me.DEFS_REDDIT.Dock = System.Windows.Forms.DockStyle.Fill
Me.DEFS_REDDIT.Location = New System.Drawing.Point(3, 3)
Me.DEFS_REDDIT.Name = "DEFS_REDDIT"
- Me.DEFS_REDDIT.RowCount = 5
+ Me.DEFS_REDDIT.RowCount = 6
+ Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEFS_REDDIT.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
- Me.DEFS_REDDIT.Size = New System.Drawing.Size(570, 352)
+ Me.DEFS_REDDIT.Size = New System.Drawing.Size(570, 366)
Me.DEFS_REDDIT.TabIndex = 1
'
'TXT_REDDIT_SAVED_POSTS_USER
'
Me.TXT_REDDIT_SAVED_POSTS_USER.CaptionText = "Saved posts user"
Me.TXT_REDDIT_SAVED_POSTS_USER.Dock = System.Windows.Forms.DockStyle.Fill
- Me.TXT_REDDIT_SAVED_POSTS_USER.Location = New System.Drawing.Point(4, 82)
+ Me.TXT_REDDIT_SAVED_POSTS_USER.Location = New System.Drawing.Point(4, 108)
Me.TXT_REDDIT_SAVED_POSTS_USER.Name = "TXT_REDDIT_SAVED_POSTS_USER"
Me.TXT_REDDIT_SAVED_POSTS_USER.Size = New System.Drawing.Size(562, 22)
- Me.TXT_REDDIT_SAVED_POSTS_USER.TabIndex = 3
+ Me.TXT_REDDIT_SAVED_POSTS_USER.TabIndex = 4
'
'TAB_DEFS_TWITTER
'
@@ -724,7 +738,7 @@
TAB_DEFS_TWITTER.Location = New System.Drawing.Point(4, 22)
TAB_DEFS_TWITTER.Name = "TAB_DEFS_TWITTER"
TAB_DEFS_TWITTER.Padding = New System.Windows.Forms.Padding(3)
- TAB_DEFS_TWITTER.Size = New System.Drawing.Size(576, 358)
+ TAB_DEFS_TWITTER.Size = New System.Drawing.Size(576, 372)
TAB_DEFS_TWITTER.TabIndex = 3
TAB_DEFS_TWITTER.Text = "Twitter"
'
@@ -743,7 +757,7 @@
Me.DEFS_TWITTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_TWITTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_TWITTER.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
- Me.DEFS_TWITTER.Size = New System.Drawing.Size(570, 352)
+ Me.DEFS_TWITTER.Size = New System.Drawing.Size(570, 366)
Me.DEFS_TWITTER.TabIndex = 1
'
'CH_TWITTER_USER_MEDIA
@@ -800,7 +814,7 @@
Me.TAB_MAIN.Location = New System.Drawing.Point(0, 0)
Me.TAB_MAIN.Name = "TAB_MAIN"
Me.TAB_MAIN.SelectedIndex = 0
- Me.TAB_MAIN.Size = New System.Drawing.Size(584, 373)
+ Me.TAB_MAIN.Size = New System.Drawing.Size(584, 398)
Me.TAB_MAIN.TabIndex = 1
'
'TAB_DEFS_INSTAGRAM
@@ -809,7 +823,7 @@
Me.TAB_DEFS_INSTAGRAM.Controls.Add(Me.DEFS_INST)
Me.TAB_DEFS_INSTAGRAM.Location = New System.Drawing.Point(4, 22)
Me.TAB_DEFS_INSTAGRAM.Name = "TAB_DEFS_INSTAGRAM"
- Me.TAB_DEFS_INSTAGRAM.Size = New System.Drawing.Size(576, 358)
+ Me.TAB_DEFS_INSTAGRAM.Size = New System.Drawing.Size(576, 372)
Me.TAB_DEFS_INSTAGRAM.TabIndex = 5
Me.TAB_DEFS_INSTAGRAM.Text = "Instagram"
'
@@ -835,7 +849,7 @@
Me.DEFS_INST.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEFS_INST.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEFS_INST.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
- Me.DEFS_INST.Size = New System.Drawing.Size(576, 358)
+ Me.DEFS_INST.Size = New System.Drawing.Size(576, 372)
Me.DEFS_INST.TabIndex = 1
'
'TXT_INST_SAVED_POSTS_USER
@@ -854,7 +868,7 @@
Me.TAB_DEFS_REDGIFS.Controls.Add(Me.DEFS_REDGIFS)
Me.TAB_DEFS_REDGIFS.Location = New System.Drawing.Point(4, 22)
Me.TAB_DEFS_REDGIFS.Name = "TAB_DEFS_REDGIFS"
- Me.TAB_DEFS_REDGIFS.Size = New System.Drawing.Size(576, 358)
+ Me.TAB_DEFS_REDGIFS.Size = New System.Drawing.Size(576, 372)
Me.TAB_DEFS_REDGIFS.TabIndex = 6
Me.TAB_DEFS_REDGIFS.Text = "RedGifs"
'
@@ -871,7 +885,7 @@
Me.DEFS_REDGIFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDGIFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEFS_REDGIFS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
- Me.DEFS_REDGIFS.Size = New System.Drawing.Size(576, 358)
+ Me.DEFS_REDGIFS.Size = New System.Drawing.Size(576, 372)
Me.DEFS_REDGIFS.TabIndex = 0
'
'CONTAINER_MAIN
@@ -880,7 +894,7 @@
'CONTAINER_MAIN.ContentPanel
'
Me.CONTAINER_MAIN.ContentPanel.Controls.Add(Me.TAB_MAIN)
- Me.CONTAINER_MAIN.ContentPanel.Size = New System.Drawing.Size(584, 373)
+ Me.CONTAINER_MAIN.ContentPanel.Size = New System.Drawing.Size(584, 398)
Me.CONTAINER_MAIN.Dock = System.Windows.Forms.DockStyle.Fill
Me.CONTAINER_MAIN.LeftToolStripPanelVisible = False
Me.CONTAINER_MAIN.Location = New System.Drawing.Point(0, 0)
@@ -890,16 +904,16 @@
Me.CONTAINER_MAIN.TabIndex = 0
Me.CONTAINER_MAIN.TopToolStripPanelVisible = False
'
- 'CH_SHOW_NOTIFY
+ 'CH_REDDIT_USER_MEDIA
'
- Me.CH_SHOW_NOTIFY.AutoSize = True
- Me.CH_SHOW_NOTIFY.Dock = System.Windows.Forms.DockStyle.Fill
- Me.CH_SHOW_NOTIFY.Location = New System.Drawing.Point(4, 318)
- Me.CH_SHOW_NOTIFY.Name = "CH_SHOW_NOTIFY"
- Me.CH_SHOW_NOTIFY.Size = New System.Drawing.Size(562, 19)
- Me.CH_SHOW_NOTIFY.TabIndex = 11
- Me.CH_SHOW_NOTIFY.Text = "Show notifications"
- Me.CH_SHOW_NOTIFY.UseVisualStyleBackColor = True
+ Me.CH_REDDIT_USER_MEDIA.AutoSize = True
+ Me.CH_REDDIT_USER_MEDIA.Dock = System.Windows.Forms.DockStyle.Fill
+ Me.CH_REDDIT_USER_MEDIA.Location = New System.Drawing.Point(4, 82)
+ Me.CH_REDDIT_USER_MEDIA.Name = "CH_REDDIT_USER_MEDIA"
+ Me.CH_REDDIT_USER_MEDIA.Size = New System.Drawing.Size(562, 19)
+ Me.CH_REDDIT_USER_MEDIA.TabIndex = 3
+ Me.CH_REDDIT_USER_MEDIA.Text = "Get user media only"
+ Me.CH_REDDIT_USER_MEDIA.UseVisualStyleBackColor = True
'
'GlobalSettingsForm
'
@@ -945,6 +959,7 @@
CType(Me.TXT_CHANNEL_USER_POST_LIMIT, System.ComponentModel.ISupportInitialize).EndInit()
TAB_DEFS_REDDIT.ResumeLayout(False)
Me.DEFS_REDDIT.ResumeLayout(False)
+ Me.DEFS_REDDIT.PerformLayout()
CType(Me.TXT_REDDIT_SAVED_POSTS_USER, System.ComponentModel.ISupportInitialize).EndInit()
TAB_DEFS_TWITTER.ResumeLayout(False)
Me.DEFS_TWITTER.ResumeLayout(False)
@@ -998,12 +1013,13 @@
Private WithEvents TXT_REQ_COUNT As PersonalUtilities.Forms.Controls.TextBoxExtended
Private WithEvents TXT_LIMIT_TIMER As PersonalUtilities.Forms.Controls.TextBoxExtended
Private WithEvents TAB_DEFS_REDGIFS As TabPage
- Friend WithEvents TAB_MAIN As TabControl
+ Private WithEvents TAB_MAIN As TabControl
Private WithEvents DEFS_TWITTER As SiteDefaults
Private WithEvents DEFS_REDDIT As SiteDefaults
Private WithEvents DEFS_INST As SiteDefaults
Private WithEvents DEFS_REDGIFS As SiteDefaults
Private WithEvents TXT_INST_SAVED_POSTS_USER As PersonalUtilities.Forms.Controls.TextBoxExtended
Private WithEvents CH_SHOW_NOTIFY As CheckBox
+ Private WithEvents CH_REDDIT_USER_MEDIA As CheckBox
End Class
End Namespace
\ No newline at end of file
diff --git a/SCrawler/Editors/GlobalSettingsForm.vb b/SCrawler/Editors/GlobalSettingsForm.vb
index 2483b40..bceabf5 100644
--- a/SCrawler/Editors/GlobalSettingsForm.vb
+++ b/SCrawler/Editors/GlobalSettingsForm.vb
@@ -91,6 +91,7 @@ Namespace Editors
'Reddit
With .Site(Sites.Reddit)
SetChecker(DEFS_REDDIT, Sites.Reddit)
+ CH_REDDIT_USER_MEDIA.Checked = .GetUserMediaOnly
TXT_REDDIT_SAVED_POSTS_USER.Text = .SavedPostsUserName
End With
'Twitter
@@ -226,6 +227,7 @@ Namespace Editors
'Reddit
With .Site(Sites.Reddit)
SetPropByChecker(DEFS_REDDIT, Sites.Reddit)
+ .GetUserMediaOnly.Value = CH_REDDIT_USER_MEDIA.Checked
.SavedPostsUserName.Value = TXT_REDDIT_SAVED_POSTS_USER.Text
End With
'Twitter
diff --git a/SCrawler/Editors/UserCreatorForm.vb b/SCrawler/Editors/UserCreatorForm.vb
index cdfbad6..45a11b4 100644
--- a/SCrawler/Editors/UserCreatorForm.vb
+++ b/SCrawler/Editors/UserCreatorForm.vb
@@ -33,11 +33,7 @@ Namespace Editors
End Property
Friend ReadOnly Property UserMediaOnly As Boolean
Get
- If User.Site = Sites.Twitter Then
- Return CH_PARSE_USER_MEDIA.Checked
- Else
- Return False
- End If
+ Return CH_PARSE_USER_MEDIA.Checked
End Get
End Property
Friend ReadOnly Property UserReady As Boolean
@@ -105,7 +101,6 @@ Namespace Editors
OPT_REDDIT.Checked = False
OPT_TWITTER.Checked = False
OPT_INSTAGRAM.Checked = False
- CH_PARSE_USER_MEDIA.Enabled = False
CH_READY_FOR_DOWN.Checked = True
CH_TEMP.Checked = Settings.DefaultTemporary
CH_DOWN_IMAGES.Checked = Settings.DefaultDownloadImages
@@ -115,7 +110,7 @@ Namespace Editors
TXT_USER.Text = User.Name
TXT_SPEC_FOLDER.Text = User.SpecialPath
Select Case User.Site
- Case Sites.Reddit : OPT_REDDIT.Checked = True : CH_PARSE_USER_MEDIA.Enabled = False
+ Case Sites.Reddit : OPT_REDDIT.Checked = True
Case Sites.Twitter : OPT_TWITTER.Checked = True
Case Sites.Instagram : OPT_INSTAGRAM.Checked = True
Case Sites.RedGifs : OPT_REDGIFS.Checked = True
@@ -212,7 +207,7 @@ Namespace Editors
Else
.Labels.ListAddList(UserLabels, LAP.NotContainsOnly, LAP.ClearBeforeAdd)
End If
- If OPT_TWITTER.Checked Then .ParseUserMediaOnly = CH_PARSE_USER_MEDIA.Checked
+ .ParseUserMediaOnly = CH_PARSE_USER_MEDIA.Checked
.UpdateUserInformation()
End With
End If
@@ -312,8 +307,7 @@ CloseForm:
CH_TEMP.Checked = .Temporary
CH_DOWN_IMAGES.Checked = .DownloadImages
CH_DOWN_VIDEOS.Checked = .DownloadVideos
- CH_PARSE_USER_MEDIA.Checked = s = Sites.Twitter AndAlso .GetUserMediaOnly.Value
- CH_PARSE_USER_MEDIA.Enabled = s = Sites.Twitter
+ CH_PARSE_USER_MEDIA.Checked = .GetUserMediaOnly.Value
CH_READY_FOR_DOWN.Checked = Not CH_TEMP.Checked
End With
End If
@@ -322,7 +316,6 @@ CloseForm:
If CH_ADD_BY_LIST.Checked Then
TXT_DESCR.GroupBoxText = "Users list"
CH_AUTO_DETECT_SITE.Enabled = True
- CH_PARSE_USER_MEDIA.Enabled = True
Else
TXT_DESCR.GroupBoxText = "Description"
CH_AUTO_DETECT_SITE.Checked = False
@@ -396,7 +389,7 @@ CloseForm:
.DownloadImages = CH_DOWN_IMAGES.Checked
.DownloadVideos = CH_DOWN_VIDEOS.Checked
.Labels.ListAddList(UserLabels)
- If s = Sites.Twitter Then .ParseUserMediaOnly = CH_PARSE_USER_MEDIA.Checked
+ .ParseUserMediaOnly = CH_PARSE_USER_MEDIA.Checked
.UpdateUserInformation()
End With
Added += 1
diff --git a/SCrawler/MainFrame.vb b/SCrawler/MainFrame.vb
index 719e0c3..9b9b49e 100644
--- a/SCrawler/MainFrame.vb
+++ b/SCrawler/MainFrame.vb
@@ -330,7 +330,8 @@ CloseResume:
If f.StartIndex >= 0 Then
OnUsersAddedHandler(f.StartIndex)
Else
- i = Settings.Users.FindIndex(Function(u) u.Site = f.User.Site And u.Name = f.User.Name)
+ Dim SimpleUser As Predicate(Of IUserData) = Function(u) u.Site = f.User.Site And u.Name = f.User.Name
+ i = Settings.Users.FindIndex(Function(u) If(u.IsCollection, DirectCast(u, UserDataBind).Collections.Exists(SimpleUser), SimpleUser.Invoke(u)))
If i < 0 Then
If Not UserBanned(f.User.Name) Then
Settings.UpdateUsersList(f.User)
diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb
index ab12c70..87473a7 100644
--- a/SCrawler/My Project/AssemblyInfo.vb
+++ b/SCrawler/My Project/AssemblyInfo.vb
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below:
'
-
-
+
+
diff --git a/SCrawler/SettingsCLS.vb b/SCrawler/SettingsCLS.vb
index 4606045..7a42ba3 100644
--- a/SCrawler/SettingsCLS.vb
+++ b/SCrawler/SettingsCLS.vb
@@ -51,11 +51,8 @@ Friend Class SettingsCLS : Implements IDisposable
Dim n() As String = {"Defaults"}
DefaultTemporary = New XMLValue(Of Boolean)("Temporary", False, MyXML, n)
- DefaultTemporary.ReplaceByValue("DefaultTemporary")
DefaultDownloadImages = New XMLValue(Of Boolean)("DownloadImages", True, MyXML, n)
- DefaultDownloadImages.ReplaceByValue("DefaultDownloadImages")
DefaultDownloadVideos = New XMLValue(Of Boolean)("DownloadVideos", True, MyXML, n)
- DefaultDownloadVideos.ReplaceByValue("DefaultDownloadVideos")
ChangeReadyForDownOnTempChange = New XMLValue(Of Boolean)("ChangeReadyForDownOnTempChange", True, MyXML, n)
MySites = New Dictionary(Of Sites, SiteSettings) From {
@@ -89,33 +86,21 @@ Friend Class SettingsCLS : Implements IDisposable
ChannelsDefaultTemporary = New XMLValue(Of Boolean)("ChannelsDefaultTemporary", True, MyXML, n)
ChannelsRegularCheckMD5 = New XMLValue(Of Boolean)("ChannelsRegularCheckMD5", False, MyXML, n)
ChannelsImagesRows = New XMLValue(Of Integer)("ImagesRows", 2, MyXML, n)
- ChannelsImagesRows.ReplaceByValue("ChannelsImagesRows")
ChannelsImagesColumns = New XMLValue(Of Integer)("ImagesColumns", 5, MyXML, n)
- ChannelsImagesColumns.ReplaceByValue("ChannelsImagesColumns")
ChannelsHideExistsUser = New XMLValue(Of Boolean)("HideExistsUser", True, MyXML, n)
- ChannelsHideExistsUser.ReplaceByValue("ChannelsHideExistsUser")
ChannelsMaxJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks, MyXML, n)
- ChannelsMaxJobsCount.ReplaceByValue("ChannelsMaxJobsCount")
n = {"Users"}
FromChannelDownloadTop = New XMLValue(Of Integer)("FromChannelDownloadTop", 10, MyXML, n)
- FromChannelDownloadTop.ReplaceByValue("FromChannelDownloadTop")
FromChannelDownloadTopUse = New XMLValue(Of Boolean)("FromChannelDownloadTopUse", False, MyXML, n)
- FromChannelDownloadTopUse.ReplaceByValue("FromChannelDownloadTopUse")
FromChannelCopyImageToUser = New XMLValue(Of Boolean)("FromChannelCopyImageToUser", True, MyXML, n)
- FromChannelCopyImageToUser.ReplaceByValue("FromChannelCopyImageToUser")
n = {"Users", "FileName"}
MaxUsersJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks, MyXML, n)
- MaxUsersJobsCount.ReplaceByValue("MaxUsersJobsCount")
FileAddDateToFileName = New XMLValue(Of Boolean)("FileAddDateToFileName", False, MyXML, n) With {.OnChangeFunction = AddressOf ChangeDateProvider}
- FileAddDateToFileName.ReplaceByValue("FileAddDateToFileName")
FileAddTimeToFileName = New XMLValue(Of Boolean)("FileAddTimeToFileName", False, MyXML, n) With {.OnChangeFunction = AddressOf ChangeDateProvider}
- FileAddTimeToFileName.ReplaceByValue("FileAddTimeToFileName")
FileDateTimePositionEnd = New XMLValue(Of Boolean)("FileDateTimePositionEnd", True, MyXML, n) With {.OnChangeFunction = AddressOf ChangeDateProvider}
- FileDateTimePositionEnd.ReplaceByValue("FileDateTimePositionEnd")
FileReplaceNameByDate = New XMLValue(Of Boolean)("FileReplaceNameByDate", False, MyXML, n)
- FileReplaceNameByDate.ReplaceByValue("FileReplaceNameByDate")
CheckUpdatesAtStart = New XMLValue(Of Boolean)("CheckUpdatesAtStart", True, MyXML)
ShowNewVersionNotification = New XMLValue(Of Boolean)("ShowNewVersionNotification", True, MyXML)
diff --git a/SCrawler/TDownloader.vb b/SCrawler/TDownloader.vb
index f5f51aa..06f85f0 100644
--- a/SCrawler/TDownloader.vb
+++ b/SCrawler/TDownloader.vb
@@ -184,8 +184,8 @@ Friend Class TDownloader : Implements IDisposable
If _Job.Count > 0 Then
Const nf As ANumbers.Formats = ANumbers.Formats.Number
Dim t As New List(Of Task)
- Dim i% = -1
- Dim j% = Settings.MaxUsersJobsCount - 1
+ Dim i% = 0
+ Dim j% = Settings.MaxUsersJobsCount
Dim limit% = IIf(_Job.Site = Sites.Instagram, 1, j)
Dim Keys As New List(Of String)
Dim h As Boolean = False