1.0.1.0

Extend settings, fix minor bugs, add some functions

Changelog.md

@@ -1,3 +1,24 @@
+# 1.0.1.0
+
+- Added
+  - Extended site settings
+  - Non-existend users will be marked in red
+  - Suspended users' profiles will be marked in yellow
+  - Automatically disable 'Ready for download' if user does not exist.
+  - Ability to disable MD5 check when downloading regular (added to the main window) channels
+  - Ability to create a user from a channel with the default option 'Ready to download' (setting in the 'Settings')
+  - Ability to change default 'Temporary' parameter on create a user from a channel (setting in the 'Settings')
+  - Advanced defaults for each site (download images, download videos and temporary)
+  - By checking the 'Temporary' checkbox in the user creation form, the 'Ready for download' checkbox became unchecked
+  - Automatically disable 'Ready for download' if profile does not exists or has been deleted
+- Change
+  - Removed extended twitter invalid credentials error and replaced with a simple line in the log
+  - Redesigned settings form
+- Fixed
+  - In some cases, the image of the channel post is not copied to the user's folder
+  - Users in the main window are not refreshed if new users are added by a list that includes banned and/or unrecognized users.
+  - Minor bugs
+
 # 1.0.0.4
 
 - Added

SCrawler/API/Base/SiteSettings.vb

@@ -60,7 +60,10 @@ Namespace API.Base
                 End If
                 Responser.SaveSettings()
             End If
-            _Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"), _XML, {Site.ToString}, XMLValue(Of SFile).ToFilePath)
+            _Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"),
+                                           _XML, {SettingsCLS.Name_Node_Sites, Site.ToString}, XMLValue(Of SFile).ToFilePath)
+            _Path.ReplaceByValue("Path", {Site.ToString})
+            _XML.Remove(Site.ToString)
         End Sub
         Friend Sub Update()
             Responser.SaveSettings()

SCrawler/API/Base/UserDataBase.vb

@@ -13,9 +13,9 @@ Imports UState = SCrawler.API.Base.UserMedia.States
 Namespace API.Base
     Friend MustInherit Class UserDataBase : Implements IUserData
         Friend Const UserFileAppender As String = "User"
-        Friend Event OnPictureUpdated As IUserData.OnPictureUpdatedEventHandler Implements IUserData.OnPictureUpdated
+        Friend Event OnUserUpdated As IUserData.OnUserUpdatedEventHandler Implements IUserData.OnUserUpdated
-        Protected Sub Raise_OnPictureUpdated()
+        Protected Sub Raise_OnUserUpdated()
-            RaiseEvent OnPictureUpdated(Me)
+            RaiseEvent OnUserUpdated(Me)
         End Sub
 #Region "Collection buttons"
         Friend WithEvents BTT_CONTEXT_DOWN As ToolStripMenuItem
@@ -43,6 +43,8 @@ Namespace API.Base
         Private Const Name_Site As String = "Site"
         Private Const Name_IsChannel As String = "IsChannel"
         Private Const Name_UserName As String = "UserName"
+        Private Const Name_UserExists As String = "UserExists"
+        Private Const Name_UserSuspended As String = "UserSuspended"
         Private Const Name_FriendlyName As String = "FriendlyName"
         Private Const Name_UserID As String = "UserID"
         Private Const Name_Description As String = "Description"
@@ -76,6 +78,10 @@ Namespace API.Base
 #Region "Declarations"
         Friend MustOverride Property Site As Sites Implements IContentProvider.Site
         Friend User As UserInfo
+        Protected Const NonExistendUserHelp As String = "404"
+        Protected Const SuspendedUserHelp As String = "403"
+        Friend Overridable Property UserExists As Boolean = True Implements IUserData.Exists
+        Friend Overridable Property UserSuspended As Boolean = False Implements IUserData.Suspended
         Friend Overridable Property Name As String Implements IContentProvider.Name
             Get
                 Return User.Name
@@ -436,10 +442,11 @@ BlockNullPicture:
                 If MyFile.Exists Then
                     FileExists = True
                     Using x As New XmlFile(MyFile) With {.XmlReadOnly = True}
-                        x.DefaultsLoading(False)
                         User.Site = Site
                         Site = x.Value(Name_Site).FromXML(Of Integer)(0)
                         User.Name = x.Value(Name_UserName)
+                        UserExists = x.Value(Name_UserExists).FromXML(Of Boolean)(True)
+                        UserSuspended = x.Value(Name_UserSuspended).FromXML(Of Boolean)(False)
                         ID = x.Value(Name_UserID)
                         FriendlyName = x.Value(Name_FriendlyName)
                         UserDescription = x.Value(Name_Description)
@@ -470,6 +477,8 @@ BlockNullPicture:
                 Using x As New XmlFile With {.Name = "User"}
                     x.Add(Name_Site, CInt(Site))
                     x.Add(Name_UserName, User.Name)
+                    x.Add(Name_UserExists, UserExists.BoolToInteger)
+                    x.Add(Name_UserSuspended, UserSuspended.BoolToInteger)
                     x.Add(Name_UserID, ID)
                     x.Add(Name_FriendlyName, FriendlyName)
                     x.Add(Name_Description, UserDescription)
@@ -508,9 +517,8 @@ BlockNullPicture:
         Private Overloads Sub LoadContentInformation(ByRef _CLIST As List(Of UserMedia), ByVal f As SFile)
             Try
                 If Not f.Exists Then Exit Sub
-                Using x As New XmlFile(f, ProtectionLevels.All, False) With {.XmlReadOnly = True, .AllowSameNames = True}
+                Using x As New XmlFile(f, Protector.Modes.All, False) With {.XmlReadOnly = True, .AllowSameNames = True}
                     x.LoadData()
-                    x.DefaultsLoading(False)
                     If x.Count > 0 Then
                         Dim fs$ = MyFile.CutPath.PathWithSeparator
                         Dim gfn As Func(Of String, String) = Function(ByVal Input As String) As String
@@ -589,11 +597,17 @@ BlockNullPicture:
 #End Region
 #Region "Download functions and options"
         Friend Overridable Property DownloadTopCount As Integer? = Nothing Implements IUserData.DownloadTopCount
+        Protected Responser As PersonalUtilities.Tools.WEB.Response
         Friend Overridable Sub DownloadData(ByVal Token As CancellationToken) Implements IContentProvider.DownloadData
             Dim Canceled As Boolean = False
             Try
                 UpdateDataFiles()
+                If Not Responser Is Nothing Then Responser.Dispose()
+                Responser = New PersonalUtilities.Tools.WEB.Response
+                Responser.Copy(Settings.Site(Site).Responser)
                 Dim UpPic As Boolean = Settings.ViewModeIsPicture AndAlso GetPicture(False) Is Nothing
+                Dim sEnvir() As Boolean = {UserExists, UserSuspended}
+                Dim EnvirChanged As Func(Of Boolean) = Function() Not sEnvir(0) = UserExists Or Not sEnvir(1) = UserSuspended
                 _DownloadedPicturesSession = 0
                 _DownloadedVideosSession = 0
                 _TempMediaList.Clear()
@@ -623,7 +637,7 @@ BlockNullPicture:
                 _ContentList.ListAddList(_ContentNew.Where(Function(c) c.State = UState.Downloaded), LNC)
                 _CountPictures = _ContentList.LongCount(Function(c) c.Type = UserMedia.Types.Picture)
                 _CountVideo = _ContentList.LongCount(Function(c) c.Type = UserMedia.Types.Video)
-                If DownloadedPictures + DownloadedVideos > 0 Then
+                If DownloadedPictures + DownloadedVideos > 0 Or EnvirChanged.Invoke Then
                     If __SaveData Then
                         LastUpdated = Now
                         If Labels.Contains(LabelsKeeper.NoParsedUser) Then Labels.Remove(LabelsKeeper.NoParsedUser)
@@ -634,12 +648,13 @@ BlockNullPicture:
                         _ContentList.Clear()
                         CreatedByChannel = False
                     End If
+                    If Not UserExists Then ReadyForDownload = False
                     UpdateUserInformation()
                 End If
                 ThrowIfDisposed()
                 _DownloadedPicturesTotal += _DownloadedPicturesSession
                 _DownloadedVideosTotal += _DownloadedVideosSession
-                If UpPic Then Raise_OnPictureUpdated()
+                If UpPic Or EnvirChanged.Invoke Then Raise_OnUserUpdated()
             Catch oex As OperationCanceledException When Token.IsCancellationRequested
                 MyMainLOG = $"{Site} - {Name}: downloading canceled"
                 Canceled = True
@@ -649,6 +664,7 @@ BlockNullPicture:
                 LogError(ex, "downloading data error")
                 HasError = True
             Finally
+                If Not Responser Is Nothing Then Responser.Dispose() : Responser = Nothing
                 If Not Canceled Then _DataParsed = True ': LastUpdated = Now
                 _ContentNew.Clear()
                 DownloadTopCount = Nothing
@@ -927,6 +943,7 @@ BlockNullPicture:
                     _ContentNew.Clear()
                     _TempMediaList.Clear()
                     _TempPostsList.Clear()
+                    If Not Responser Is Nothing Then Responser.Dispose()
                     If Not BTT_CONTEXT_DOWN Is Nothing Then BTT_CONTEXT_DOWN.Dispose()
                     If Not BTT_CONTEXT_EDIT Is Nothing Then BTT_CONTEXT_EDIT.Dispose()
                     If Not BTT_CONTEXT_DELETE Is Nothing Then BTT_CONTEXT_DELETE.Dispose()
@@ -959,7 +976,7 @@ BlockNullPicture:
         Sub DownloadData(ByVal Token As CancellationToken)
     End Interface
     Friend Interface IUserData : Inherits IContentProvider, IComparable(Of UserDataBase), IComparable, IEquatable(Of UserDataBase), IDisposable
-        Event OnPictureUpdated(ByVal User As IUserData)
+        Event OnUserUpdated(ByVal User As IUserData)
         Property ParseUserMediaOnly As Boolean
 #Region "Images"
         Function GetPicture() As Image
@@ -972,6 +989,8 @@ BlockNullPicture:
         ReadOnly Property Labels As List(Of String)
 #End Region
         ReadOnly Property IsChannel As Boolean
+        Property Exists As Boolean
+        Property Suspended As Boolean
         Property ReadyForDownload As Boolean
         Property [File] As SFile
         Property FileExists As Boolean

SCrawler/API/Reddit/Channel.vb

@@ -22,9 +22,23 @@ Namespace API.Reddit
 #End Region
         Friend Const DefaultDownloadLimitCount As Integer = 1000
 #Region "IUserData Support"
-        Private Event OnPictureUpdated(User As IUserData) Implements IUserData.OnPictureUpdated
+        Private Event OnUserUpdated As IUserData.OnUserUpdatedEventHandler Implements IUserData.OnUserUpdated
         Friend Property Instance As IUserData
         Private Property IUserData_ParseUserMediaOnly As Boolean = False Implements IUserData.ParseUserMediaOnly
+        Private Property IUserData_Exists As Boolean Implements IUserData.Exists
+            Get
+                Return Instance.Exists
+            End Get
+            Set(ByVal e As Boolean)
+            End Set
+        End Property
+        Private Property IUserData_Suspended As Boolean Implements IUserData.Suspended
+            Get
+                Return Instance.Suspended
+            End Get
+            Set(ByVal s As Boolean)
+            End Set
+        End Property
         Private ReadOnly Property IUserData_IsCollection As Boolean Implements IUserData.IsCollection
             Get
                 Return Instance.IsCollection
@@ -504,9 +518,8 @@ Namespace API.Reddit
         End Function
         Friend Overloads Function LoadData(ByVal f As SFile, ByVal PartialLoad As Boolean, Optional ByVal e As ErrorsDescriber = Nothing) As Boolean
             If f.Exists Then
-                Using x As New XmlFile(f, ProtectionLevels.All, False) With {.XmlReadOnly = True, .AllowSameNames = True}
+                Using x As New XmlFile(f, Protector.Modes.All, False) With {.XmlReadOnly = True, .AllowSameNames = True}
                     x.LoadData()
-                    x.DefaultsLoading(False)
                     If x.Count > 0 Then
                         Dim XMLDateProvider As New ADateTime(ADateTime.Formats.BaseDateTime)
                         Name = x.Value(Name_Name)
@@ -527,7 +540,6 @@ Namespace API.Reddit
         Friend Overloads Function Save(Optional ByVal f As SFile = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Boolean Implements ILoaderSaver.Save
             Dim XMLDateProvider As New ADateTime(ADateTime.Formats.BaseDateTime)
             Using x As New XmlFile With {.AllowSameNames = True, .Name = "Channel"}
-                x.DefaultsLoading(False)
                 x.Add(Name_Name, Name)
                 x.Add(Name_ID, ID)
                 If Posts.Count > 0 Or PostsLatest.Count > 0 Then

SCrawler/API/Reddit/UserData.vb

@@ -62,6 +62,8 @@ Namespace API.Reddit
 #Region "Initializers"
         ''' <summary>Video downloader initializer</summary>
         Private Sub New()
+            ChannelPostsNames = New List(Of String)
+            _ExistsUsersNames = New List(Of String)
         End Sub
         ''' <summary>Default initializer</summary>
         Friend Sub New(ByVal u As UserInfo, Optional ByVal _LoadUserInformation As Boolean = True, Optional ByVal InvokeImageHandler As Boolean = True)
@@ -75,6 +77,9 @@ Namespace API.Reddit
 #Region "Download Overrides"
         Friend Overrides Sub DownloadData(ByVal Token As CancellationToken)
             If IsChannel AndAlso Not ChannelInfo.IsRegularChannel Then
+                If Not Responser Is Nothing Then Responser.Dispose()
+                Responser = New PersonalUtilities.Tools.WEB.Response
+                Responser.Copy(Settings.Site(Sites.Reddit).Responser)
                 ChannelPostsNames.ListAddList(ChannelInfo.PostsAll.Select(Function(p) p.ID), LNC)
                 If SkipExistsUsers Then _ExistsUsersNames.ListAddList(Settings.UsersList.Select(Function(p) p.Name), LNC)
                 DownloadDataF(Token)
@@ -201,8 +206,14 @@ Namespace API.Reddit
             Catch oex As OperationCanceledException When Token.IsCancellationRequested
             Catch dex As ObjectDisposedException When Disposed
             Catch ex As Exception
+                If ex.HelpLink = NonExistendUserHelp Then
+                    UserExists = False
+                ElseIf ex.HelpLink = SuspendedUserHelp Then
+                    UserSuspended = True
+                Else
                     LogError(ex, $"data downloading error [{URL}]")
                     HasError = True
+                End If
             End Try
         End Sub
         Private Sub DownloadDataChannel(ByVal POST As String, ByVal Token As CancellationToken)
@@ -283,8 +294,14 @@ Namespace API.Reddit
             Catch oex As OperationCanceledException When Token.IsCancellationRequested
             Catch dex As ObjectDisposedException When Disposed
             Catch ex As Exception
+                If ex.HelpLink = NonExistendUserHelp Then
+                    UserExists = False
+                ElseIf ex.HelpLink = SuspendedUserHelp Then
+                    UserSuspended = True
+                Else
                     LogError(ex, $"channel data downloading error [{URL}]")
                     HasError = True
+                End If
             End Try
         End Sub
 #End Region
@@ -384,6 +401,8 @@ Namespace API.Reddit
                 If Not URL.IsEmptyString AndAlso URL.Contains("redgifs") Then
                     Using r As New UserData
                         r._TempMediaList.Add(MediaFromData(UTypes.VideoPre, URL, String.Empty, String.Empty,, False))
+                        r.Responser = New PersonalUtilities.Tools.WEB.Response
+                        r.Responser.Copy(Settings.Site(Sites.Reddit).Responser)
                         r.ReparseVideo(Nothing)
                         If r._TempMediaList.ListExists Then Return r._TempMediaList(0)
                     End Using
@@ -442,6 +461,7 @@ Namespace API.Reddit
                         Dim cached As Boolean = IsChannel And SaveToCache
                         Dim vsf As Boolean = SeparateVideoFolderF
                         Dim ImgFormat As Imaging.ImageFormat
+                        Dim UseMD5 As Boolean = Not IsChannel Or (Not cached And Settings.ChannelsRegularCheckMD5)
                         Dim bDP As New ErrorsDescriber(EDP.None)
                         Dim MD5BS As Func(Of String, UTypes,
                                              SFile, Boolean, String) = Function(ByVal __URL As String, ByVal __MT As UTypes,
@@ -474,7 +494,7 @@ Namespace API.Reddit
                                 End If
                                 f.Separator = "\"
                                 m = String.Empty
-                                If (v.Type = UTypes.Picture Or v.Type = UTypes.GIF) And Not cached Then
+                                If (v.Type = UTypes.Picture Or v.Type = UTypes.GIF) And UseMD5 Then
                                     m = MD5BS(v.URL, v.Type, f, False)
                                     If m.IsEmptyString AndAlso Not v.URL_BASE.IsEmptyString AndAlso Not v.URL_BASE = v.URL Then
                                         m = MD5BS(v.URL_BASE, v.Type, f, True)
@@ -483,7 +503,7 @@ Namespace API.Reddit
                                 End If
 
                                 If (Not m.IsEmptyString AndAlso Not HashList.Contains(m)) Or Not (v.Type = UTypes.Picture Or
-                                                                                                  v.Type = UTypes.GIF) Or cached Then
+                                                                                                  v.Type = UTypes.GIF) Or Not UseMD5 Then
                                     If Not cached Then HashList.Add(m)
                                     v.MD5 = m
                                     f.Path = MyDir
@@ -536,15 +556,20 @@ Namespace API.Reddit
         End Sub
         Protected Function GetSiteResponse(ByVal URL As String, Optional ByVal e As ErrorsDescriber = Nothing) As String
             Try
-                Return Settings.Site(Sites.Reddit).Responser.GetResponse(URL,, EDP.ThrowException)
+                Return Responser.GetResponse(URL,, EDP.ThrowException)
             Catch ex As Exception
                 HasError = True
                 Dim OptText$ = String.Empty
                 If Not e.Exists Then
                     Dim ee As EDP = EDP.SendInLog
-                    If Settings.Site(Sites.Reddit).Responser.StatusCode = HttpStatusCode.NotFound Then
+                    If Responser.StatusCode = HttpStatusCode.NotFound Then
-                        ee += EDP.ThrowException
+                        ee = EDP.ThrowException
                         OptText = ": USER NOT FOUND"
+                        ex.HelpLink = NonExistendUserHelp
+                    ElseIf Responser.StatusCode = HttpStatusCode.Forbidden Then
+                        ee = EDP.ThrowException
+                        OptText = ": USER PROFILE SUSPENDED"
+                        ex.HelpLink = SuspendedUserHelp
                     Else
                         ee += EDP.ReturnValue
                     End If

SCrawler/API/Twitter/UserData.vb

@@ -6,6 +6,7 @@
 '
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
+Imports PersonalUtilities.Tools.WEB
 Imports PersonalUtilities.Tools.WebDocuments.JSON
 Imports PersonalUtilities.Functions.XML
 Imports System.Net
@@ -67,7 +68,7 @@ Namespace API.Twitter
                 If Not POST.IsEmptyString Then URL &= $"&max_id={POST}"
 
                 ThrowAny(Token)
-                Dim r$ = Settings.Site(Sites.Twitter).Responser.GetResponse(URL,, EDP.ThrowException)
+                Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException)
                 If Not r.IsEmptyString Then
                     Using w As EContainer = JsonDocument.Parse(r)
                         If Not w Is Nothing AndAlso w.Count > 0 Then
@@ -117,8 +118,16 @@ Namespace API.Twitter
             Catch oex As OperationCanceledException When Token.IsCancellationRequested
             Catch dex As ObjectDisposedException When Disposed
             Catch ex As Exception
+                If Responser.StatusCode = HttpStatusCode.NotFound Then
+                    UserExists = False
+                ElseIf Responser.StatusCode = HttpStatusCode.Unauthorized Then
+                    UserSuspended = True
+                ElseIf Responser.StatusCode = HttpStatusCode.BadRequest Then
+                    MyMainLOG = "Twitter has invalid credentials"
+                Else
                     LogError(ex, $"data downloading error [{URL}]")
                     HasError = True
+                End If
             End Try
         End Sub
         Friend Shared Function GetVideoInfo(ByVal URL As String) As UserMedia
@@ -126,8 +135,8 @@ Namespace API.Twitter
                 If URL.Contains("twitter") Then
                     Dim PostID$ = RegexReplace(URL, New RegexStructure("(?<=/)\d+", True, False,,,,, String.Empty))
                     If Not PostID.IsEmptyString Then
-                        Dim r$ = Settings.Site(Sites.Twitter).Responser.GetResponse($"https://api.twitter.com/1.1/statuses/show.json?id={PostID}",,
+                        Dim r$ = DirectCast(Settings.Site(Sites.Twitter).Responser.Copy(), Response).
-                                                                                    EDP.ReturnValue)
+                                            GetResponse($"https://api.twitter.com/1.1/statuses/show.json?id={PostID}",, EDP.ReturnValue)
                         If Not r.IsEmptyString Then
                             Using j As EContainer = JsonDocument.Parse(r)
                                 If j.ListExists Then

SCrawler/API/UserDataBind.vb

@@ -36,6 +36,20 @@ Namespace API
                 CollectionName = NewCollectionName
             End Set
         End Property
+        Friend Overrides Property UserExists As Boolean
+            Get
+                Return Count > 0 AndAlso Collections.Exists(Function(c) c.Exists)
+            End Get
+            Set(ByVal e As Boolean)
+            End Set
+        End Property
+        Friend Overrides Property UserSuspended As Boolean
+            Get
+                Return Count > 0 AndAlso Collections.LongCount(Function(c) c.Suspended) = Count
+            End Get
+            Set(ByVal s As Boolean)
+            End Set
+        End Property
         Friend Overrides Sub ChangeCollectionName(ByVal NewName As String, ByVal UpdateSettings As Boolean)
             _CollectionName = NewName
             If Count > 0 Then Collections.ForEach(Sub(c) c.CollectionName = NewName)
@@ -278,8 +292,8 @@ Namespace API
         End Sub
         Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
         End Sub
-        Private Sub User_OnPictureUpdated(ByVal User As IUserData)
+        Private Sub User_OnUserUpdated(ByVal User As IUserData)
-            Raise_OnPictureUpdated()
+            Raise_OnUserUpdated()
         End Sub
         Friend Overrides Sub OpenSite()
             If Count > 0 Then Collections(0).OpenSite()
@@ -314,7 +328,7 @@ Namespace API
                             .UpdateUserInformation()
                         End If
                         ImageHandler(_Item, False)
-                        AddHandler .Self.OnPictureUpdated, AddressOf User_OnPictureUpdated
+                        AddHandler .Self.OnUserUpdated, AddressOf User_OnUserUpdated
                         DirectCast(.Self, UserDataBase).CreateButtons(Count - 1)
                     End With
                 Else
@@ -333,7 +347,7 @@ Namespace API
                 .CreateButtons(Count - 1)
                 AddHandler .BTT_CONTEXT_DELETE.Click, AddressOf BTT_CONTEXT_DELETE_Click
             End With
-            AddHandler Collections(Count - 1).OnPictureUpdated, AddressOf User_OnPictureUpdated
+            AddHandler Collections.Last.OnUserUpdated, AddressOf User_OnUserUpdated
         End Sub
         Friend Sub AddRange(ByVal _Items As IEnumerable(Of IUserData))
             If Not _Items Is Nothing AndAlso _Items.Count > 0 Then

SCrawler/Channels/ChannelViewForm.vb

@@ -424,7 +424,11 @@ Friend Class ChannelViewForm : Implements IChannelLimits
                         If Not Settings.UsersList.Contains(.ElementAt(i)) Then
                             f = PendingUsers(i).File
                             Settings.UpdateUsersList(.ElementAt(i))
-                            Settings.Users.Add(New UserData(.ElementAt(i), False) With {.Temporary = True, .CreatedByChannel = True})
+                            Settings.Users.Add(New UserData(.ElementAt(i), False) With {
+                                               .Temporary = Settings.ChannelsDefaultTemporary,
+                                               .CreatedByChannel = True,
+                                               .ReadyForDownload = Settings.ChannelsDefaultReadyForDownload
+                                               })
                             With Settings.Users.Last
                                 .Labels.Add(CannelsLabelName)
                                 .UpdateUserInformation()
@@ -568,6 +572,7 @@ Friend Class ChannelViewForm : Implements IChannelLimits
     End Sub
     Private Sub ChangeComboIndex(ByVal Appender As Integer)
         Try
+            AppendPendingUsers()
             Dim _ComboUpEnabled As Boolean = False
             Dim _ComboDownEnabled As Boolean = False
             If CMB_CHANNELS.Count > 0 Then

SCrawler/Editors/GlobalSettingsForm.resx

@@ -117,7 +117,7 @@
   <resheader name="writer">
     <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
   </resheader>
-  <metadata name="TP_MAIN.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+  <metadata name="TP_BASIS.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
     <value>False</value>
   </metadata>
   <assembly alias="System.Drawing" name="System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a" />
@@ -140,17 +140,9 @@
         AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC
 </value>
   </data>
-  <metadata name="TT_MAIN.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+  <metadata name="TP_IMAGES.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
     <value>False</value>
   </metadata>
-  <metadata name="TT_MAIN.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
-    <value>17, 17</value>
-  </metadata>
-  <data name="CH_SEPARATE_VIDEO_FOLDER.ToolTip" xml:space="preserve">
-    <value>This is global setting for new added only users.
-This parameter indicates how video will be stored in users' download path.
-If checked then videos will be stored in separated folder; otherwise, videos will be stored with pictures together.</value>
-  </data>
   <data name="ActionButton3.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
     <value>
         iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
@@ -159,12 +151,6 @@ If checked then videos will be stored in separated folder; otherwise, videos wil
         AFuc5QFgn6ClHh5iOQVAKNixyucB8NY0vG9JOzzyhrdq5IRgAAAAAElFTkSuQmCC
 </value>
   </data>
-  <metadata name="TP_IMAGES.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
-    <value>False</value>
-  </metadata>
-  <metadata name="TP_CHANNELS_IMGS.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
-    <value>False</value>
-  </metadata>
   <data name="ActionButton4.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
     <value>
         iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6
@@ -197,9 +183,17 @@ If checked then videos will be stored in separated folder; otherwise, videos wil
         VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg==
 </value>
   </data>
-  <metadata name="TP_DEFAULTS.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+  <metadata name="TT_MAIN.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
     <value>False</value>
   </metadata>
+  <metadata name="TT_MAIN.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
+    <value>17, 17</value>
+  </metadata>
+  <data name="CH_SEPARATE_VIDEO_FOLDER.ToolTip" xml:space="preserve">
+    <value>This is a global setting for newly added users only.
+This parameter specifies how the video will be stored in the users' download path.
+If checked, videos will be stored in separate folder; otherwise, videos will be stored along with images.</value>
+  </data>
   <metadata name="TP_FILE_NAME.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
     <value>False</value>
   </metadata>
@@ -209,4 +203,37 @@ If checked then videos will be stored in separated folder; otherwise, videos wil
   <metadata name="LBL_DATE_POS.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
     <value>False</value>
   </metadata>
+  <metadata name="TP_CHANNELS_IMGS.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="TAB_MAIN.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="TAB_BASIS.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="TAB_DEFAULTS.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="TP_DEFS.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="TAB_DEFS_CHANNELS.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="TP_CHANNELS.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="TAB_DEFS_REDDIT.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="TP_REDDIT.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="TAB_DEFS_TWITTER.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
+  <metadata name="TP_TWITTER.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
+    <value>False</value>
+  </metadata>
 </root>

SCrawler/Editors/GlobalSettingsForm.vb

@@ -23,23 +23,27 @@ Namespace Editors
                     .AddOkCancelToolbar()
                     .DelegateClosingChecker()
                     With Settings
+                        'Basis
                         TXT_GLOBAL_PATH.Text = .GlobalPath.Value
                         TXT_IMAGE_LARGE.Value = .MaxLargeImageHeigh.Value
                         TXT_IMAGE_SMALL.Value = .MaxSmallImageHeigh.Value
                         TXT_COLLECTIONS_PATH.Text = .CollectionsPath
+                        TXT_MAX_JOBS_USERS.Value = .MaxUsersJobsCount.Value
+                        TXT_MAX_JOBS_CHANNELS.Value = .ChannelsMaxJobsCount.Value
+                        CH_CHECK_VER_START.Checked = .CheckUpdatesAtStart
+                        'Defaults
                         CH_SEPARATE_VIDEO_FOLDER.Checked = .SeparateVideoFolder.Value
                         CH_DEF_TEMP.Checked = .DefaultTemporary
                         CH_DOWN_IMAGES.Checked = .DefaultDownloadImages
                         CH_DOWN_VIDEOS.Checked = .DefaultDownloadVideos
-                        TXT_CHANNELS_COLUMNS.Value = .ChannelsImagesColumns.Value
+                        'Channels
                         TXT_CHANNELS_ROWS.Value = .ChannelsImagesRows.Value
+                        TXT_CHANNELS_COLUMNS.Value = .ChannelsImagesColumns.Value
                         TXT_CHANNEL_USER_POST_LIMIT.Value = .FromChannelDownloadTop.Value
                         TXT_CHANNEL_USER_POST_LIMIT.Checked = .FromChannelDownloadTopUse.Value
                         CH_COPY_CHANNEL_USER_IMAGE.Checked = .FromChannelCopyImageToUser
-                        CH_CHECK_VER_START.Checked = .CheckUpdatesAtStart
+                        CH_CHANNELS_USERS_TEMP.Checked = .ChannelsDefaultTemporary
-                        TXT_MAX_JOBS_USERS.Value = .MaxUsersJobsCount.Value
+                        'Channels filenames
-                        TXT_MAX_JOBS_CHANNELS.Value = .ChannelsMaxJobsCount.Value
-
                         CH_FILE_NAME_CHANGE.Checked = .FileReplaceNameByDate Or .FileAddDateToFileName Or .FileAddTimeToFileName
                         OPT_FILE_NAME_REPLACE.Checked = .FileReplaceNameByDate
                         OPT_FILE_NAME_ADD_DATE.Checked = Not .FileReplaceNameByDate
@@ -47,6 +51,15 @@ Namespace Editors
                         CH_FILE_TIME.Checked = .FileAddTimeToFileName
                         OPT_FILE_DATE_START.Checked = Not .FileDateTimePositionEnd
                         OPT_FILE_DATE_END.Checked = .FileDateTimePositionEnd
+                        'Reddit
+                        SetChecker(CH_REDDIT_TEMP, .RedditTemporary)
+                        SetChecker(CH_REDDIT_DOWN_IMG, .RedditDownloadImages)
+                        SetChecker(CH_REDDIT_DOWN_VID, .RedditDownloadVideos)
+                        'Twitter
+                        SetChecker(CH_TWITTER_TEMP, .TwitterTemporary)
+                        SetChecker(CH_TWITTER_DOWN_IMG, .TwitterDownloadImages)
+                        SetChecker(CH_TWITTER_DOWN_VID, .TwitterDownloadVideos)
+                        CH_TWITTER_USER_MEDIA.Checked = .TwitterDefaultGetUserMedia
                     End With
                     .MyFieldsChecker = New FieldsChecker
                     With .MyFieldsChecker
@@ -62,6 +75,20 @@ Namespace Editors
                 MyDefs.InvokeLoaderError(ex)
             End Try
         End Sub
+        Private Sub SetChecker(ByRef CH As CheckBox, ByVal Prop As XML.Base.XMLValue(Of Boolean))
+            If Prop.ValueF.Exists Then
+                CH.Checked = Prop.Value
+            Else
+                CH.CheckState = CheckState.Indeterminate
+            End If
+        End Sub
+        Private Sub SetPropByChecker(ByRef Prop As XML.Base.XMLValue(Of Boolean), ByRef CH As CheckBox)
+            Select Case CH.CheckState
+                Case CheckState.Checked : Prop.Value = True
+                Case CheckState.Unchecked : Prop.Value = False
+                Case CheckState.Indeterminate : Prop.ValueF = Nothing
+            End Select
+        End Sub
         Private Sub ToolbarBttOK() Implements IOkCancelToolbar.ToolbarBttOK
             If MyDefs.MyFieldsChecker.AllParamsOK Then
                 With Settings
@@ -93,21 +120,26 @@ Namespace Editors
                     End If
 
                     .BeginUpdate()
+                    'Basis
                     .GlobalPath.Value = TXT_GLOBAL_PATH.Text
                     .MaxLargeImageHeigh.Value = CInt(TXT_IMAGE_LARGE.Value)
                     .MaxSmallImageHeigh.Value = CInt(TXT_IMAGE_SMALL.Value)
-                    .SeparateVideoFolder.Value = CH_SEPARATE_VIDEO_FOLDER.Checked
                     .CollectionsPath.Value = TXT_COLLECTIONS_PATH.Text
+                    .MaxUsersJobsCount.Value = CInt(TXT_MAX_JOBS_USERS.Value)
+                    .ChannelsMaxJobsCount.Value = TXT_MAX_JOBS_CHANNELS.Value
+                    .CheckUpdatesAtStart.Value = CH_CHECK_VER_START.Checked
+                    'Defaults
+                    .SeparateVideoFolder.Value = CH_SEPARATE_VIDEO_FOLDER.Checked
                     .DefaultTemporary.Value = CH_DEF_TEMP.Checked
                     .DefaultDownloadImages.Value = CH_DOWN_IMAGES.Checked
                     .DefaultDownloadVideos.Value = CH_DOWN_VIDEOS.Checked
+                    'Channels
                     .ChannelsImagesRows.Value = CInt(TXT_CHANNELS_ROWS.Value)
                     .ChannelsImagesColumns.Value = CInt(TXT_CHANNELS_COLUMNS.Value)
-                    .FromChannelDownloadTopUse.Value = TXT_CHANNEL_USER_POST_LIMIT.Checked
                     .FromChannelDownloadTop.Value = CInt(TXT_CHANNEL_USER_POST_LIMIT.Value)
+                    .FromChannelDownloadTopUse.Value = TXT_CHANNEL_USER_POST_LIMIT.Checked
                     .FromChannelCopyImageToUser.Value = CH_COPY_CHANNEL_USER_IMAGE.Checked
-                    .CheckUpdatesAtStart.Value = CH_CHECK_VER_START.Checked
+                    .ChannelsDefaultTemporary.Value = CH_CHANNELS_USERS_TEMP.Checked
-                    .MaxUsersJobsCount.Value = CInt(TXT_MAX_JOBS_USERS.Value)
 
                     If CH_FILE_NAME_CHANGE.Checked Then
                         .FileReplaceNameByDate.Value = OPT_FILE_NAME_REPLACE.Checked
@@ -119,6 +151,15 @@ Namespace Editors
                         .FileAddTimeToFileName.Value = False
                         .FileReplaceNameByDate.Value = False
                     End If
+                    'Reddit
+                    SetPropByChecker(.RedditTemporary, CH_REDDIT_TEMP)
+                    SetPropByChecker(.RedditDownloadImages, CH_REDDIT_DOWN_IMG)
+                    SetPropByChecker(.RedditDownloadVideos, CH_REDDIT_DOWN_VID)
+                    'Twitter
+                    SetPropByChecker(.TwitterTemporary, CH_TWITTER_TEMP)
+                    SetPropByChecker(.TwitterDownloadImages, CH_TWITTER_DOWN_IMG)
+                    SetPropByChecker(.TwitterDownloadVideos, CH_TWITTER_DOWN_VID)
+                    .TwitterDefaultGetUserMedia.Value = CH_TWITTER_USER_MEDIA.Checked
 
                     .EndUpdate()
                 End With
@@ -140,13 +181,13 @@ Namespace Editors
         Private Sub TXT_MAX_JOBS_CHANNELS_ActionOnButtonClick(ByVal Sender As ActionButton) Handles TXT_MAX_JOBS_CHANNELS.ActionOnButtonClick
             If Sender.DefaultButton = ActionButton.DefaultButtons.Refresh Then TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks
         End Sub
-        Private Sub CH_FILE_NAME_CHANGE_CheckedChanged(sender As Object, e As EventArgs) Handles CH_FILE_NAME_CHANGE.CheckedChanged
+        Private Sub CH_FILE_NAME_CHANGE_CheckedChanged(sender As Object, e As EventArgs)
             ChangeFileNameChangersEnabling()
         End Sub
-        Private Sub OPT_FILE_NAME_REPLACE_CheckedChanged(sender As Object, e As EventArgs) Handles OPT_FILE_NAME_REPLACE.CheckedChanged
+        Private Sub OPT_FILE_NAME_REPLACE_CheckedChanged(sender As Object, e As EventArgs)
             ChangePositionControlsEnabling()
         End Sub
-        Private Sub OPT_FILE_NAME_ADD_DATE_CheckedChanged(sender As Object, e As EventArgs) Handles OPT_FILE_NAME_ADD_DATE.CheckedChanged
+        Private Sub OPT_FILE_NAME_ADD_DATE_CheckedChanged(sender As Object, e As EventArgs)
             ChangePositionControlsEnabling()
         End Sub
         Private Sub ChangePositionControlsEnabling()

SCrawler/Editors/UserCreatorForm.vb

@@ -122,8 +122,10 @@ Namespace Editors
                                 UserLabels.ListAddList(.Labels)
                                 If UserLabels.ListExists Then TXT_LABELS.Text = UserLabels.ListToString
                             End With
+                            CH_ADD_BY_LIST.Enabled = False
                         Else
-                            CH_READY_FOR_DOWN.Checked = Settings.DefaultTemporary
+                            CH_TEMP.Checked = Settings.DefaultTemporary
+                            CH_READY_FOR_DOWN.Checked = Not Settings.DefaultTemporary
                             CH_DOWN_IMAGES.Checked = Settings.DefaultDownloadImages
                             CH_DOWN_VIDEOS.Checked = Settings.DefaultDownloadVideos
                         End If
@@ -131,6 +133,7 @@ Namespace Editors
                     .MyFieldsChecker = New FieldsChecker
                     .MyFieldsChecker.AddControl(Of String)(TXT_USER, TXT_USER.CaptionText)
                     .MyFieldsChecker.EndLoaderOperations()
+                    SetParamsBySite()
                     .AppendDetectors()
                     .EndLoaderOperations()
                 End With
@@ -146,7 +149,6 @@ Namespace Editors
                 e.Cancel = True
             Else
                 MyDef.Dispose()
-                If Not DialogResult = DialogResult.OK And Not DialogResult = DialogResult.Cancel And StartIndex >= 0 Then DialogResult = DialogResult.OK
             End If
         End Sub
         Private Sub UserCreatorForm_Disposed(sender As Object, e As EventArgs) Handles Me.Disposed
@@ -242,26 +244,50 @@ CloseForm:
             If Not s.IsEmptyString Then TXT = s : Return True Else Return False
         End Function
         Private Sub OPT_REDDIT_CheckedChanged(sender As Object, e As EventArgs) Handles OPT_REDDIT.CheckedChanged
-            If OPT_REDDIT.Checked Then CH_IS_CHANNEL.Enabled = True
+            If OPT_REDDIT.Checked Then CH_IS_CHANNEL.Enabled = True : SetParamsBySite()
         End Sub
         Private Sub OPT_TWITTER_CheckedChanged(sender As Object, e As EventArgs) Handles OPT_TWITTER.CheckedChanged
-            CH_PARSE_USER_MEDIA.Enabled = OPT_TWITTER.Checked
+            If OPT_TWITTER.Checked Then CH_IS_CHANNEL.Checked = False : CH_IS_CHANNEL.Enabled = False : SetParamsBySite()
-            If OPT_TWITTER.Checked Then CH_IS_CHANNEL.Checked = False : CH_IS_CHANNEL.Enabled = False
         End Sub
         Private Sub CH_TEMP_CheckedChanged(sender As Object, e As EventArgs) Handles CH_TEMP.CheckedChanged
-            If CH_TEMP.Checked Then CH_FAV.Checked = False
+            If CH_TEMP.Checked Then CH_FAV.Checked = False : CH_READY_FOR_DOWN.Checked = False
         End Sub
         Private Sub CH_FAV_CheckedChanged(sender As Object, e As EventArgs) Handles CH_FAV.CheckedChanged
             If CH_FAV.Checked Then CH_TEMP.Checked = False
         End Sub
+        Private Sub SetParamsBySite()
+            Dim s As Sites = Sites.Undefined
+            Select Case True
+                Case OPT_REDDIT.Checked : s = Sites.Reddit
+                Case OPT_TWITTER.Checked : s = Sites.Twitter
+            End Select
+            With Settings
+                Select Case s
+                    Case Sites.Reddit
+                        CH_TEMP.Checked = .RedditTemporary
+                        CH_DOWN_IMAGES.Checked = .RedditDownloadImages
+                        CH_DOWN_VIDEOS.Checked = .RedditDownloadVideos
+                        CH_PARSE_USER_MEDIA.Checked = False
+                        CH_PARSE_USER_MEDIA.Enabled = False
+                    Case Sites.Twitter
+                        CH_TEMP.Checked = .TwitterTemporary
+                        CH_DOWN_IMAGES.Checked = .TwitterDownloadImages
+                        CH_DOWN_VIDEOS.Checked = .TwitterDownloadVideos
+                        CH_PARSE_USER_MEDIA.Enabled = True
+                        CH_PARSE_USER_MEDIA.Checked = .TwitterDefaultGetUserMedia
+                End Select
+            End With
+        End Sub
         Private Sub CH_ADD_BY_LIST_CheckedChanged(sender As Object, e As EventArgs) Handles CH_ADD_BY_LIST.CheckedChanged
             If CH_ADD_BY_LIST.Checked Then
                 TXT_DESCR.GroupBoxText = "Users list"
                 CH_AUTO_DETECT_SITE.Enabled = True
+                CH_PARSE_USER_MEDIA.Enabled = True
             Else
                 TXT_DESCR.GroupBoxText = "Description"
                 CH_AUTO_DETECT_SITE.Checked = False
                 CH_AUTO_DETECT_SITE.Enabled = False
+                SetParamsBySite()
             End If
             TXT_USER.Enabled = Not CH_ADD_BY_LIST.Checked
             TXT_USER_FRIENDLY.Enabled = Not CH_ADD_BY_LIST.Checked
@@ -355,6 +381,7 @@ CloseForm:
                             End If
 
                             MsgBoxE(m)
+                            If Added > 0 Then MyDef.ChangesDetected = False
                             Return Added > 0 And Not BannedUsers.ListExists And NonIdentified.Count = 0
                         Else
                             MsgBoxE("No one user can not recognized", MsgBoxStyle.Exclamation)

SCrawler/MainFrame.vb

@@ -170,6 +170,17 @@ CloseResume:
                                 Case View.SmallIcon : .SmallImageList.Images.Add(User.LVIKey, User.GetPicture())
                             End Select
                             .Items.Add(User.GetLVI(LIST_PROFILES))
+                            If Not User.Exists Then
+                                With .Items(.Items.Count - 1)
+                                    .BackColor = ColorBttDeleteBack
+                                    .ForeColor = ColorBttDeleteFore
+                                End With
+                            ElseIf User.Suspended Then
+                                With .Items(.Items.Count - 1)
+                                    .BackColor = ColorBttEditBack
+                                    .ForeColor = ColorBttEditFore
+                                End With
+                            End If
                         End With
                     End Sub
             Else
@@ -188,6 +199,16 @@ CloseResume:
                                 End Select
                                 .Items(i).Text = User.ToString
                                 .Items(i).Group = User.GetLVIGroup(LIST_PROFILES)
+                                If Not User.Exists Then
+                                    .Items(i).BackColor = ColorBttDeleteBack
+                                    .Items(i).ForeColor = ColorBttDeleteFore
+                                ElseIf User.Suspended Then
+                                    .Items(i).BackColor = ColorBttEditBack
+                                    .Items(i).ForeColor = ColorBttEditFore
+                                Else
+                                    .Items(i).BackColor = SystemColors.Window
+                                    .Items(i).ForeColor = SystemColors.WindowText
+                                End If
                             End If
                         End With
                     End Sub
@@ -239,7 +260,7 @@ CloseResume:
     Private Sub BTT_ADD_USER_Click(sender As Object, e As EventArgs) Handles BTT_ADD_USER.Click
         Using f As New UserCreatorForm
             f.ShowDialog()
-            If f.DialogResult = DialogResult.OK Then
+            If f.DialogResult = DialogResult.OK Or f.StartIndex >= 0 Then
                 Dim i%
                 If f.StartIndex >= 0 Then
                     OnUsersAddedHandler(f.StartIndex)
@@ -818,7 +839,7 @@ ResumeDownloadingOperation:
         If Not user Is Nothing Then user.OpenFolder()
     End Sub
 #End Region
-    Friend Sub User_OnPictureUpdated(ByVal User As IUserData)
+    Friend Sub User_OnUserUpdated(ByVal User As IUserData)
         UserListUpdate(User, False)
     End Sub
     Private _LogVisible As Boolean = False

SCrawler/MainMod.vb

@@ -13,7 +13,7 @@ Friend Module MainMod
     Friend Settings As SettingsCLS
     Friend Const SettingsFolderName As String = "Settings"
     Friend ReadOnly LinkPattern As New RegexStructure("[htps:]{0,6}[/]{0,2}(.+)", 1)
-    Friend ReadOnly FilesPattern As New RegexStructure("[^\./]+?\.\w+", True, False, 1)
+    Friend ReadOnly FilesPattern As New RegexStructure("[^\./]+?\.\w+", True, False, 1,,,, String.Empty, EDP.ReturnValue)
     Friend Const LVI_TempOption As String = "Temp"
     Friend Const LVI_FavOption As String = "Favorite"
     Friend Const CannelsLabelName As String = "Channels"
@@ -171,9 +171,9 @@ Friend Module MainMod
     Friend Sub ImageHandler(ByVal User As IUserData, ByVal Add As Boolean)
         Try
             If Add Then
-                AddHandler User.Self.OnPictureUpdated, AddressOf MainFrame.User_OnPictureUpdated
+                AddHandler User.Self.OnUserUpdated, AddressOf MainFrame.User_OnUserUpdated
             Else
-                RemoveHandler User.Self.OnPictureUpdated, AddressOf MainFrame.User_OnPictureUpdated
+                RemoveHandler User.Self.OnUserUpdated, AddressOf MainFrame.User_OnUserUpdated
             End If
         Catch ex As Exception
         End Try

SCrawler/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("1.0.0.4")>
+<Assembly: AssemblyVersion("1.0.1.0")>
-<Assembly: AssemblyFileVersion("1.0.0.4")>
+<Assembly: AssemblyFileVersion("1.0.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/SettingsCLS.vb

@@ -12,6 +12,7 @@ Imports SCrawler.API
 Imports SCrawler.API.Base
 Friend Class SettingsCLS : Implements IDisposable
     Friend Const DefaultMaxDownloadingTasks As Integer = 5
+    Friend Const Name_Node_Sites As String = "Sites"
     Friend ReadOnly Design As XmlFile
     Private ReadOnly MyXML As XmlFile
     Friend ReadOnly OS64 As Boolean
@@ -35,8 +36,7 @@ Friend Class SettingsCLS : Implements IDisposable
         FfmpegFile = "ffmpeg.exe"
         FfmpegExists = FfmpegFile.Exists
         If OS64 And Not FfmpegExists Then MsgBoxE("[ffmpeg.exe] is missing", vbExclamation)
-        Design = New XmlFile("Settings\Design.xml")
+        Design = New XmlFile("Settings\Design.xml", Protector.Modes.All)
-        Design.DefaultsLoading(False)
         MyXML = New XmlFile(Nothing) With {.AutoUpdateFile = True}
         Users = New List(Of IUserData)
         UsersList = New List(Of UserInfo)
@@ -52,16 +52,43 @@ Friend Class SettingsCLS : Implements IDisposable
 
         SeparateVideoFolder = New XMLValue(Of Boolean)("SeparateVideoFolder", True, MyXML)
         CollectionsPath = New XMLValue(Of String)("CollectionsPath", "Collections", MyXML)
-        MaxUsersJobsCount = New XMLValue(Of Integer)("MaxUsersJobsCount", DefaultMaxDownloadingTasks, MyXML)
 
-        DefaultTemporary = New XMLValue(Of Boolean)("DefaultTemporary", False, MyXML)
+        Dim n() As String = {"Defaults"}
-        DefaultDownloadImages = New XMLValue(Of Boolean)("DefaultDownloadImages", True, MyXML)
+        DefaultTemporary = New XMLValue(Of Boolean)("Temporary", False, MyXML, n)
-        DefaultDownloadVideos = New XMLValue(Of Boolean)("DefaultDownloadVideos", True, MyXML)
+        DefaultTemporary.ReplaceByValue("DefaultTemporary")
+        DefaultDownloadImages = New XMLValue(Of Boolean)("DownloadImages", True, MyXML, n)
+        DefaultDownloadImages.ReplaceByValue("DefaultDownloadImages")
+        DefaultDownloadVideos = New XMLValue(Of Boolean)("DownloadVideos", True, MyXML, n)
+        DefaultDownloadVideos.ReplaceByValue("DefaultDownloadVideos")
+        ChangeReadyForDownOnTempChange = New XMLValue(Of Boolean)("ChangeReadyForDownOnTempChange", True, MyXML, n)
 
-        FileAddDateToFileName = New XMLValue(Of Boolean)("FileAddDateToFileName", False, MyXML) With {.OnChangeFunction = AddressOf ChangeDateProvider}
+        n = {Name_Node_Sites, Sites.Reddit.ToString}
-        FileAddTimeToFileName = New XMLValue(Of Boolean)("FileAddTimeToFileName", False, MyXML) With {.OnChangeFunction = AddressOf ChangeDateProvider}
+        RedditTemporary = New XMLValue(Of Boolean)
-        FileDateTimePositionEnd = New XMLValue(Of Boolean)("FileDateTimePositionEnd", True, MyXML) With {.OnChangeFunction = AddressOf ChangeDateProvider}
+        RedditTemporary.SetExtended("Temporary", False, MyXML, n)
-        FileReplaceNameByDate = New XMLValue(Of Boolean)("FileReplaceNameByDate", False, MyXML)
+        RedditTemporary.SetDefault(DefaultTemporary)
+
+        RedditDownloadImages = New XMLValue(Of Boolean)
+        RedditDownloadImages.SetExtended("DownloadImages", True, MyXML, n)
+        RedditDownloadImages.SetDefault(DefaultDownloadImages)
+
+        RedditDownloadVideos = New XMLValue(Of Boolean)
+        RedditDownloadVideos.SetExtended("DownloadVideos", True, MyXML, n)
+        RedditDownloadVideos.SetDefault(DefaultDownloadVideos)
+
+        n = {Name_Node_Sites, Sites.Twitter.ToString}
+        TwitterTemporary = New XMLValue(Of Boolean)
+        TwitterTemporary.SetExtended("Temporary", False, MyXML, n)
+        TwitterTemporary.SetDefault(DefaultTemporary)
+
+        TwitterDownloadImages = New XMLValue(Of Boolean)
+        TwitterDownloadImages.SetExtended("DownloadImages", True, MyXML, n)
+        TwitterDownloadImages.SetDefault(DefaultDownloadImages)
+
+        TwitterDownloadVideos = New XMLValue(Of Boolean)
+        TwitterDownloadVideos.SetExtended("DownloadVideos", True, MyXML, n)
+        TwitterDownloadVideos.SetDefault(DefaultDownloadVideos)
+
+        TwitterDefaultGetUserMedia = New XMLValue(Of Boolean)("TwitterDefaultGetUserMedia", True, MyXML, n)
 
         MaxLargeImageHeigh = New XMLValue(Of Integer)("MaxLargeImageHeigh", 150, MyXML)
         MaxSmallImageHeigh = New XMLValue(Of Integer)("MaxSmallImageHeigh", 15, MyXML)
@@ -73,26 +100,51 @@ Friend Class SettingsCLS : Implements IDisposable
         LatestSelectedLabels = New XMLValue(Of String)("LatestSelectedLabels",, MyXML)
         LatestSelectedChannel = New XMLValue(Of String)("LatestSelectedChannel",, MyXML)
 
-        ChannelsImagesRows = New XMLValue(Of Integer)("ChannelsImagesRows", 2, MyXML)
+        n = {Name_Node_Sites, "Channels"}
-        ChannelsImagesColumns = New XMLValue(Of Integer)("ChannelsImagesColumns", 5, MyXML)
+        ChannelsDefaultReadyForDownload = New XMLValue(Of Boolean)("ChannelsDefaultReadyForDownload", False, MyXML, n)
-        ChannelsHideExistsUser = New XMLValue(Of Boolean)("ChannelsHideExistsUser", True, MyXML)
+        ChannelsDefaultTemporary = New XMLValue(Of Boolean)("ChannelsDefaultTemporary", True, MyXML, n)
-        ChannelsMaxJobsCount = New XMLValue(Of Integer)("ChannelsMaxJobsCount", DefaultMaxDownloadingTasks, MyXML)
+        ChannelsRegularCheckMD5 = New XMLValue(Of Boolean)("ChannelsRegularCheckMD5", False, MyXML, n)
-        FromChannelDownloadTop = New XMLValue(Of Integer)("FromChannelDownloadTop", 10, MyXML)
+        ChannelsImagesRows = New XMLValue(Of Integer)("ImagesRows", 2, MyXML, n)
-        FromChannelDownloadTopUse = New XMLValue(Of Boolean)("FromChannelDownloadTopUse", False, MyXML)
+        ChannelsImagesRows.ReplaceByValue("ChannelsImagesRows")
-        FromChannelCopyImageToUser = New XMLValue(Of Boolean)("FromChannelCopyImageToUser", True, MyXML)
+        ChannelsImagesColumns = New XMLValue(Of Integer)("ImagesColumns", 5, MyXML, n)
+        ChannelsImagesColumns.ReplaceByValue("ChannelsImagesColumns")
+        ChannelsHideExistsUser = New XMLValue(Of Boolean)("HideExistsUser", True, MyXML, n)
+        ChannelsHideExistsUser.ReplaceByValue("ChannelsHideExistsUser")
+        ChannelsMaxJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks, MyXML, n)
+        ChannelsMaxJobsCount.ReplaceByValue("ChannelsMaxJobsCount")
+
+        n = {"Users"}
+        FromChannelDownloadTop = New XMLValue(Of Integer)("FromChannelDownloadTop", 10, MyXML, n)
+        FromChannelDownloadTop.ReplaceByValue("FromChannelDownloadTop")
+        FromChannelDownloadTopUse = New XMLValue(Of Boolean)("FromChannelDownloadTopUse", False, MyXML, n)
+        FromChannelDownloadTopUse.ReplaceByValue("FromChannelDownloadTopUse")
+        FromChannelCopyImageToUser = New XMLValue(Of Boolean)("FromChannelCopyImageToUser", True, MyXML, n)
+        FromChannelCopyImageToUser.ReplaceByValue("FromChannelCopyImageToUser")
+
+        n = {"Users", "FileName"}
+        MaxUsersJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks, MyXML, n)
+        MaxUsersJobsCount.ReplaceByValue("MaxUsersJobsCount")
+        FileAddDateToFileName = New XMLValue(Of Boolean)("FileAddDateToFileName", False, MyXML, n) With {.OnChangeFunction = AddressOf ChangeDateProvider}
+        FileAddDateToFileName.ReplaceByValue("FileAddDateToFileName")
+        FileAddTimeToFileName = New XMLValue(Of Boolean)("FileAddTimeToFileName", False, MyXML, n) With {.OnChangeFunction = AddressOf ChangeDateProvider}
+        FileAddTimeToFileName.ReplaceByValue("FileAddTimeToFileName")
+        FileDateTimePositionEnd = New XMLValue(Of Boolean)("FileDateTimePositionEnd", True, MyXML, n) With {.OnChangeFunction = AddressOf ChangeDateProvider}
+        FileDateTimePositionEnd.ReplaceByValue("FileDateTimePositionEnd")
+        FileReplaceNameByDate = New XMLValue(Of Boolean)("FileReplaceNameByDate", False, MyXML, n)
+        FileReplaceNameByDate.ReplaceByValue("FileReplaceNameByDate")
 
         CheckUpdatesAtStart = New XMLValue(Of Boolean)("CheckUpdatesAtStart", True, MyXML)
         ShowNewVersionNotification = New XMLValue(Of Boolean)("ShowNewVersionNotification", True, MyXML)
         LatestVersion = New XMLValue(Of String)("LatestVersion", String.Empty, MyXML)
 
-        MyXML.DefaultsLoading(False)
+        If MyXML.ChangesDetected Then MyXML.Sort() : MyXML.UpdateData()
-        If MyXML.ChangesDetected Then MyXML.UpdateData()
         Labels = New LabelsKeeper
         If Not LatestSelectedLabels.IsEmptyString Then Labels.CurrentSelection.ListAddList(LatestSelectedLabels.Value.StringToList(Of String, List(Of String))("|"))
         If BlackListFile.Exists Then
             BlackList.ListAddList(IO.File.ReadAllLines(BlackListFile), LAP.NotContainsOnly)
             If BlackList.Count > 0 Then BlackList.RemoveAll(Function(b) Not b.Exists)
         End If
+        _UpdatesSuspended = False
         ChangeDateProvider(Nothing, Nothing, Nothing)
     End Sub
     Private Sub ChangeDateProvider(ByVal Sender As Object, ByVal Name As String, ByVal Value As Object)
@@ -108,9 +160,8 @@ Friend Class SettingsCLS : Implements IDisposable
         Try
             Users.Clear()
             If UsersSettingsFile.Exists Then
-                Using x As New XmlFile(UsersSettingsFile, ProtectionLevels.All, False) With {.AllowSameNames = True}
+                Using x As New XmlFile(UsersSettingsFile, Protector.Modes.All, False) With {.AllowSameNames = True}
                     x.LoadData()
-                    x.DefaultsLoading(False)
                     If x.Count > 0 Then x.ForEach(Sub(xx) UsersList.Add(xx))
                 End Using
                 Dim PNC As Func(Of UserInfo, Boolean) = Function(u) Not u.IncludedInCollection
@@ -184,6 +235,12 @@ Friend Class SettingsCLS : Implements IDisposable
         Catch ex As Exception
         End Try
     End Sub
+    Private _UserListUpdateRequired As Boolean = False
+    Friend ReadOnly Property UserListUpdateRequired As Boolean
+        Get
+            Return _UserListUpdateRequired
+        End Get
+    End Property
     Friend Overloads Sub UpdateUsersList(ByVal u As UserInfo)
         Dim i% = UsersList.IndexOf(u)
         If i >= 0 Then
@@ -194,13 +251,17 @@ Friend Class SettingsCLS : Implements IDisposable
         UpdateUsersList()
     End Sub
     Friend Overloads Sub UpdateUsersList()
+        Try
             If UsersList.Count > 0 Then
                 Using x As New XmlFile With {.AllowSameNames = True, .Name = "Users"}
-                x.DefaultsLoading(False)
                     UsersList.ForEach(Sub(u) x.Add(u.GetContainer()))
                     x.Save(UsersSettingsFile)
                 End Using
             End If
+            _UserListUpdateRequired = False
+        Catch ex As Exception
+            _UserListUpdateRequired = True
+        End Try
     End Sub
     Friend Sub UpdateBlackList()
         If BlackList.Count > 0 Then
@@ -228,7 +289,7 @@ Friend Class SettingsCLS : Implements IDisposable
     Friend Overloads Function UserExists(ByVal _User As UserInfo) As Boolean
         Return UserExists(_User.Site, _User.Name)
     End Function
-    Private _UpdatesSuspended As Boolean = False
+    Private _UpdatesSuspended As Boolean = True
     Friend Sub BeginUpdate()
         MyXML.BeginUpdate()
         _UpdatesSuspended = True
@@ -261,13 +322,30 @@ Friend Class SettingsCLS : Implements IDisposable
     Friend ReadOnly Property DefaultTemporary As XMLValue(Of Boolean)
     Friend ReadOnly Property DefaultDownloadImages As XMLValue(Of Boolean)
     Friend ReadOnly Property DefaultDownloadVideos As XMLValue(Of Boolean)
+    Friend ReadOnly Property ChangeReadyForDownOnTempChange As XMLValue(Of Boolean)
+#Region "Reddit"
+    Friend ReadOnly Property RedditTemporary As XMLValue(Of Boolean)
+    Friend ReadOnly Property RedditDownloadImages As XMLValue(Of Boolean)
+    Friend ReadOnly Property RedditDownloadVideos As XMLValue(Of Boolean)
 #End Region
-#Region "Additional info"
+#Region "Twitter"
+    Friend ReadOnly Property TwitterTemporary As XMLValue(Of Boolean)
+    Friend ReadOnly Property TwitterDownloadImages As XMLValue(Of Boolean)
+    Friend ReadOnly Property TwitterDownloadVideos As XMLValue(Of Boolean)
+    Friend ReadOnly Property TwitterDefaultGetUserMedia As XMLValue(Of Boolean)
+#End Region
+#End Region
+#Region "User data"
+    Friend ReadOnly Property FromChannelDownloadTop As XMLValue(Of Integer)
+    Friend ReadOnly Property FromChannelDownloadTopUse As XMLValue(Of Boolean)
+    Friend ReadOnly Property FromChannelCopyImageToUser As XMLValue(Of Boolean)
+#Region "File naming"
     Friend ReadOnly Property FileAddDateToFileName As XMLValue(Of Boolean)
     Friend ReadOnly Property FileAddTimeToFileName As XMLValue(Of Boolean)
     Friend ReadOnly Property FileDateTimePositionEnd As XMLValue(Of Boolean)
     Friend ReadOnly Property FileReplaceNameByDate As XMLValue(Of Boolean)
 #End Region
+#End Region
 #Region "View"
     Friend ReadOnly Property MaxLargeImageHeigh As XMLValue(Of Integer)
     Friend ReadOnly Property MaxSmallImageHeigh As XMLValue(Of Integer)
@@ -289,13 +367,13 @@ Friend Class SettingsCLS : Implements IDisposable
     Friend ReadOnly Property LatestSelectedChannel As XMLValue(Of String)
 #End Region
 #Region "Channels properties"
+    Friend ReadOnly Property ChannelsDefaultReadyForDownload As XMLValue(Of Boolean)
+    Friend ReadOnly Property ChannelsDefaultTemporary As XMLValue(Of Boolean)
+    Friend ReadOnly Property ChannelsRegularCheckMD5 As XMLValue(Of Boolean)
     Friend ReadOnly Property ChannelsImagesRows As XMLValue(Of Integer)
     Friend ReadOnly Property ChannelsImagesColumns As XMLValue(Of Integer)
     Friend ReadOnly Property ChannelsHideExistsUser As XMLValue(Of Boolean)
     Friend ReadOnly Property ChannelsMaxJobsCount As XMLValue(Of Integer)
-    Friend ReadOnly Property FromChannelDownloadTop As XMLValue(Of Integer)
-    Friend ReadOnly Property FromChannelDownloadTopUse As XMLValue(Of Boolean)
-    Friend ReadOnly Property FromChannelCopyImageToUser As XMLValue(Of Boolean)
 #End Region
 #Region "New version properties"
     Friend ReadOnly Property CheckUpdatesAtStart As XMLValue(Of Boolean)
@@ -307,6 +385,7 @@ Friend Class SettingsCLS : Implements IDisposable
     Protected Overridable Overloads Sub Dispose(ByVal disposing As Boolean)
         If Not disposedValue Then
             If disposing Then
+                If UserListUpdateRequired Then UpdateUsersList()
                 If Not Channels Is Nothing Then
                     Channels.Dispose()
                     If Reddit.ChannelsCollection.ChannelsPathCache.Exists(SFO.Path, False) Then _

SCrawler/TDownloader.vb

@@ -136,6 +136,10 @@ Friend Class TDownloader : Implements IDisposable
         Catch ex As Exception
             ErrorsDescriber.Execute(EDP.SendInLog, ex, "TDownloader.DownloadData")
         Finally
+            If Settings.UserListUpdateRequired Then _
+                Task.WaitAll(Task.Run(Sub()
+                                          While Settings.UserListUpdateRequired : Settings.UpdateUsersList() : End While
+                                      End Sub))
             MainProgress.Enabled(EOptions.ProgressBar) = False
         End Try
     End Sub