Added plugins: LPSG, XVIDEOS
Updated hosts
Deleted IResponserContainer
Updated SiteSettingsForm (properties, responser, button, cookies, offset)
Updated plugin dependencies
Fixed Instagram algo typo
This commit is contained in:
Andy
2022-03-22 20:14:17 +03:00
parent 2be52c7c56
commit fccb786e55
53 changed files with 2703 additions and 121 deletions

View File

@@ -11,11 +11,12 @@ Imports PersonalUtilities.Functions.RegularExpressions
Imports PersonalUtilities.Tools.WEB
Imports Download = SCrawler.Plugin.ISiteSettings.Download
Namespace API.Base
Friend MustInherit Class SiteSettingsBase : Implements ISiteSettings, IResponserContainer
Friend MustInherit Class SiteSettingsBase : Implements ISiteSettings
Friend ReadOnly Property Site As String Implements ISiteSettings.Site
Friend Overridable ReadOnly Property Icon As Icon = Nothing Implements ISiteSettings.Icon
Friend Overridable ReadOnly Property Image As Image = Nothing Implements ISiteSettings.Image
Friend Overridable ReadOnly Property Responser As Response Implements IResponserContainer.Responser
Private Property Logger As ILogProvider = LogConnector Implements ISiteSettings.Logger
Friend Overridable ReadOnly Property Responser As Response
Friend MustOverride Function GetInstance(ByVal What As Download) As IPluginContentProvider Implements ISiteSettings.GetInstance
Friend Sub New(ByVal SiteName As String)
Site = SiteName
@@ -82,7 +83,7 @@ Namespace API.Base
Return Nothing
End If
End Function
Friend Overridable Function GetSpecialData(ByVal URL As String) As IEnumerable(Of IPluginUserMedia) Implements ISiteSettings.GetSpecialData
Friend Overridable Function GetSpecialData(ByVal URL As String, ByVal Path As String, ByVal AskForPath As Boolean) As IEnumerable(Of PluginUserMedia) Implements ISiteSettings.GetSpecialData
Return Nothing
End Function
Friend Overridable Function GetSpecialDataF(ByVal URL As String) As IEnumerable(Of UserMedia)

View File

@@ -42,7 +42,7 @@ Namespace API.Base
Me.New(_URL)
[Type] = _Type
End Sub
Friend Sub New(ByVal m As Plugin.IPluginUserMedia)
Friend Sub New(ByVal m As Plugin.PluginUserMedia)
If Not IsNothing(m) Then
[Type] = m.ContentType
URL = m.URL

View File

@@ -9,6 +9,7 @@
Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.RegularExpressions
Imports PersonalUtilities.Forms.Toolbars
Imports PersonalUtilities.Tools.WEB
Imports System.IO
Imports System.Net
Imports System.Threading
@@ -421,7 +422,7 @@ BlockNullPicture:
Friend Property ExternalPlugin As IPluginContentProvider
Private Property IPluginContentProvider_ExistingContentList As List(Of PluginUserMedia) Implements IPluginContentProvider.ExistingContentList
Private Property IPluginContentProvider_TempPostsList As List(Of String) Implements IPluginContentProvider.TempPostsList
Private Property IPluginContentProvider_TempMediaList As List(Of IPluginUserMedia) Implements IPluginContentProvider.TempMediaList
Private Property IPluginContentProvider_TempMediaList As List(Of PluginUserMedia) Implements IPluginContentProvider.TempMediaList
Private Property IPluginContentProvider_SeparateVideoFolder As Boolean Implements IPluginContentProvider.SeparateVideoFolder
Private Property IPluginContentProvider_DataPath As String Implements IPluginContentProvider.DataPath
Private Sub IPluginContentProvider_XmlFieldsSet(ByVal Fields As List(Of KeyValuePair(Of String, String))) Implements IPluginContentProvider.XmlFieldsSet
@@ -712,7 +713,7 @@ BlockNullPicture:
#Region "Download functions and options"
Friend Overridable Property DownloadTopCount As Integer? = Nothing Implements IUserData.DownloadTopCount, IPluginContentProvider.PostsNumberLimit
Friend Overridable Property DownloadToDate As Date? = Nothing Implements IUserData.DownloadToDate, IPluginContentProvider.PostsDateLimit
Protected Responser As PersonalUtilities.Tools.WEB.Response
Protected Responser As Response
Friend Overridable Sub DownloadData(ByVal Token As CancellationToken) Implements IContentProvider.DownloadData
Dim Canceled As Boolean = False
_ExternalCompatibilityToken = Token
@@ -720,12 +721,8 @@ BlockNullPicture:
UpdateDataFiles()
UserDescriptionReset()
If Not Responser Is Nothing Then Responser.Dispose()
Responser = New PersonalUtilities.Tools.WEB.Response
If TypeOf HOST.Source Is IResponserContainer Then
With DirectCast(HOST.Source, IResponserContainer)
If Not .Responser Is Nothing Then Responser.Copy(.Responser)
End With
End If
Responser = New Response
If Not HOST.Responser Is Nothing Then Responser.Copy(HOST.Responser)
Dim UpPic As Boolean = Settings.ViewModeIsPicture AndAlso GetPicture(False) Is Nothing
Dim sEnvir() As Boolean = {UserExists, UserSuspended}

View File

@@ -260,7 +260,7 @@ Namespace API.Instagram
If Not TaggedChecked Then
TaggedCount = j.Value("total_count").FromXML(Of Integer)(0)
TaggedChecked = True
If TaggedChecked > 200 Then
If TaggedCount > 200 Then
Dim a% = MsgBoxE({$"The number of tagged posts is {TaggedCount.NumToString(New ANumbers With {
.FormatOptions = ANumbers.Options.GroupIntegral})}" & vbCr &
"The tagged data download operation can take a long time.", "Too much tagged data"}, vbExclamation,,,

View File

@@ -19,7 +19,7 @@ Namespace Editors
Private ReadOnly LBL_AUTH As Label
Private ReadOnly LBL_OTHER As Label
Private ReadOnly MyDefs As DefaultFormProps(Of FieldsChecker)
Private SpecialButton As Button
Private WithEvents SpecialButton As Button
#Region "Providers"
Private Class SavedPostsChecker : Implements ICustomProvider
Private Function Convert(ByVal Value As Object, ByVal DestinationType As Type, ByVal Provider As IFormatProvider,
@@ -70,24 +70,27 @@ Namespace Editors
.AddControl(Of String)(TXT_PATH_SAVED_POSTS, TXT_PATH_SAVED_POSTS.CaptionText, True, New SavedPostsChecker)
End With
Dim offset% = PropertyValueHost.LeftOffsetDefault
Dim h% = 0, c% = 0
Dim AddTpControl As Action(Of Control, Integer) = Sub(ByVal cnt As Control, ByVal _height As Integer)
TP_SITE_PROPS.RowStyles.Add(New RowStyle(SizeType.Absolute, _height))
TP_SITE_PROPS.RowCount += 1
TP_SITE_PROPS.Controls.Add(cnt, 0, TP_SITE_PROPS.RowStyles.Count - 1)
h += _height
c += 1
End Sub
If Host.Responser Is Nothing Then
h -= 28
TXT_COOKIES.Enabled = False
TXT_COOKIES.Visible = False
TP_MAIN.RowStyles(2).Height = 0
End If
If .PropList.Count > 0 Then
Dim offset% = PropertyValueHost.LeftOffsetDefault
Dim h% = 0, c% = 0
Dim laAdded As Boolean = False
Dim loAdded As Boolean = False
If Not Host.IsMyClass Then
h -= 28
TXT_COOKIES.Enabled = False
TXT_COOKIES.Visible = False
TP_MAIN.RowStyles(2).Height = 0
End If
Dim AddTpControl As Action(Of Control, Integer) = Sub(ByVal cnt As Control, ByVal _height As Integer)
TP_SITE_PROPS.RowStyles.Add(New RowStyle(SizeType.Absolute, _height))
TP_SITE_PROPS.RowCount += 1
TP_SITE_PROPS.Controls.Add(cnt, 0, TP_SITE_PROPS.RowStyles.Count - 1)
h += _height
c += 1
End Sub
Dim pArr() As Boolean
If .PropList.Exists(Function(p) If(p.Options?.IsAuth, False)) Then pArr = {True, False} Else pArr = {False}
.PropList.Sort()
@@ -117,20 +120,21 @@ Namespace Editors
End If
Next
Next
SpecialButton = .GetSettingsButtonInternal
If Not SpecialButton Is Nothing Then AddTpControl(SpecialButton, 28)
TP_SITE_PROPS.BaseControlsPadding = New Padding(offset, 0, 0, 0)
offset += PaddingE.GetOf({TP_SITE_PROPS}).Left
TXT_PATH.CaptionWidth = offset
TXT_PATH_SAVED_POSTS.CaptionWidth = offset
TXT_COOKIES.CaptionWidth = offset
CH_GET_USER_MEDIA_ONLY.Padding = New PaddingE(CH_GET_USER_MEDIA_ONLY.Padding) With {.Left = offset}
If c > 0 Or Not Host.IsMyClass Then
Dim ss As New Size(Size.Width, Size.Height + h + c)
MinimumSize = ss
Size = ss
MaximumSize = ss
End If
End If
SpecialButton = .GetSettingsButtonInternal
If Not SpecialButton Is Nothing Then AddTpControl(SpecialButton, 28)
TP_SITE_PROPS.BaseControlsPadding = New Padding(offset, 0, 0, 0)
offset += PaddingE.GetOf({TP_SITE_PROPS}).Left
TXT_PATH.CaptionWidth = offset
TXT_PATH_SAVED_POSTS.CaptionWidth = offset
TXT_COOKIES.CaptionWidth = offset
CH_GET_USER_MEDIA_ONLY.Padding = New PaddingE(CH_GET_USER_MEDIA_ONLY.Padding) With {.Left = offset}
If c > 0 Or Not Host.IsMyClass Then
Dim ss As New Size(Size.Width, Size.Height + h + c)
MinimumSize = ss
Size = ss
MaximumSize = ss
End If
End With
@@ -208,8 +212,8 @@ Namespace Editors
End Sub
Private Sub TXT_COOKIES_ActionOnButtonClick(ByVal Sender As ActionButton) Handles TXT_COOKIES.ActionOnButtonClick
If Sender.DefaultButton = ActionButton.DefaultButtons.Edit Then
If TypeOf Host.Source Is IResponserContainer Then
Using f As New CookieListForm(DirectCast(Host.Source, IResponserContainer).Responser) With {
If Not Host.Responser Is Nothing Then
Using f As New CookieListForm(Host.Responser) With {
.MyDesignXML = Settings.Design,
.DisableControls = CookieControl.AddFromInternal + CookieControl.AuthorizeProgram + CookieControl.OpenBrowser
}
@@ -220,8 +224,8 @@ Namespace Editors
End If
End Sub
Private Sub TXT_COOKIES_ActionOnButtonClearClick() Handles TXT_COOKIES.ActionOnButtonClearClick
If TypeOf Host.Source Is IResponserContainer Then
With DirectCast(Host.Source, IResponserContainer).Responser
If Not Host.Responser Is Nothing Then
With Host.Responser
If Not .Cookies Is Nothing Then .Cookies.Dispose()
.Cookies = New CookieKeeper(.CookiesDomain)
End With
@@ -229,8 +233,10 @@ Namespace Editors
End If
End Sub
Private Sub SetCookieText()
If TypeOf Host.Source Is IResponserContainer Then _
TXT_COOKIES.Text = $"{If(DirectCast(Host.Source, IResponserContainer).Responser.Cookies?.Count, 0)} cookies"
If Not Host.Responser Is Nothing Then TXT_COOKIES.Text = $"{If(Host.Responser.Cookies?.Count, 0)} cookies"
End Sub
Private Sub SpecialButton_Click(sender As Object, e As EventArgs) Handles SpecialButton.Click
MyDefs.Detector()
End Sub
End Class
End Namespace

View File

@@ -292,8 +292,13 @@ Friend Module MainMod
For Each p As PluginHost In Settings.Plugins
d = p.Settings.IsMyImageVideo(URL)
If d.Exists Then
um = Settings(d.HostKey).GetSpecialData(URL)
um = Settings(d.HostKey).GetSpecialData(URL, Settings.LatestSavingPath.Value, AskForPath)
found = True
If um.ListExists Then
If AskForPath And Not um(0).SpecialFolder.IsEmptyString And Not p.Settings.IsMyClass Then _
Settings.LatestSavingPath.Value = um(0).SpecialFolder
If um(0).State = UserMedia.States.Downloaded Then Return True
End If
Exit For
End If
Next

View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("3.0.0.1")>
<Assembly: AssemblyFileVersion("3.0.0.1")>
<Assembly: AssemblyVersion("3.0.0.2")>
<Assembly: AssemblyFileVersion("3.0.0.2")>
<Assembly: NeutralResourcesLanguage("en")>

View File

@@ -1,13 +0,0 @@
' Copyright (C) 2022 Andy
' This program is free software: you can redistribute it and/or modify
' it under the terms of the GNU General Public License as published by
' the Free Software Foundation, either version 3 of the License, or
' (at your option) any later version.
'
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Namespace Plugin
Friend Interface IResponserContainer
ReadOnly Property Responser As PersonalUtilities.Tools.WEB.Response
End Interface
End Namespace

View File

@@ -11,6 +11,7 @@ Imports SCrawler.API.Base
Imports SCrawler.Plugin.Attributes
Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Functions.XML.Base
Imports PersonalUtilities.Tools.WEB
Imports Download = SCrawler.Plugin.ISiteSettings.Download
Namespace Plugin.Hosts
Friend Class SettingsHost
@@ -25,7 +26,7 @@ Namespace Plugin.Hosts
End Function
Friend Function GetSettingsButtonInternal() As Button
If Not SpecialFormAttribute Is Nothing AndAlso SpecialFormAttribute.SettingsForm Then
BTT_SETTINGS_INTERNAL = New Button With {.Text = "Other settings", .Dock = DockStyle.Right}
BTT_SETTINGS_INTERNAL = New Button With {.Text = "Other settings", .Dock = DockStyle.Right, .Width = 150}
Return BTT_SETTINGS_INTERNAL
Else
Return Nothing
@@ -77,6 +78,16 @@ Namespace Plugin.Hosts
End Get
End Property
Friend ReadOnly Property HasSpecialOptions As Boolean = False
Private ReadOnly _ResponserGetMethod As MethodInfo
Friend ReadOnly Property Responser As Response
Get
If Not _ResponserGetMethod Is Nothing Then
Return _ResponserGetMethod.Invoke(Source, Nothing)
Else
Return Nothing
End If
End Get
End Property
#End Region
#Region "Base properties compatibility"
Friend ReadOnly Property Temporary As XMLValue(Of Boolean)
@@ -128,6 +139,7 @@ Namespace Plugin.Hosts
Friend Sub New(ByVal Plugin As ISiteSettings, ByRef _XML As XmlFile, ByVal GlobalPath As SFile,
ByRef _Temp As XMLValue(Of Boolean), ByRef _Imgs As XMLValue(Of Boolean), ByRef _Vids As XMLValue(Of Boolean))
Source = Plugin
Source.Logger = LogConnector
PropList = New List(Of PropertyValueHost)
@@ -162,7 +174,7 @@ Namespace Plugin.Hosts
Source.BeginInit()
Dim n() As String = {SettingsCLS.Name_Node_Sites, Name}
If If(_XML(n)?.Count, 0) > 0 Then Source.Load(_XML(n).ToKeyValuePair)
If If(_XML(n)?.Count, 0) > 0 Then Source.Load(ToKeyValuePair(Of String, EContainer)(_XML(n)))
Dim Members As IEnumerable(Of MemberInfo) = Plugin.GetType.GetTypeInfo.DeclaredMembers
If Members.ListExists Then
Dim Updaters As New List(Of MemberInfo)
@@ -170,7 +182,12 @@ Namespace Plugin.Hosts
Dim PropCheckers As New List(Of MemberInfo)
Dim m As MemberInfo
For Each m In Members
If m.MemberType = MemberTypes.Property Then PropList.Add(New PropertyValueHost(Source, m))
If m.MemberType = MemberTypes.Property Then
PropList.Add(New PropertyValueHost(Source, m))
With DirectCast(m, PropertyInfo)
If .PropertyType Is GetType(Response) Then _ResponserGetMethod = .GetMethod
End With
End If
With m.GetCustomAttributes()
If .ListExists Then
If m.MemberType = MemberTypes.Method Then
@@ -265,14 +282,14 @@ Namespace Plugin.Hosts
If s.Exists Then s.SiteName = Name : s.HostKey = Key
Return s
End Function
Friend Function GetSpecialData(ByVal URL As String) As IEnumerable(Of UserMedia)
Friend Function GetSpecialData(ByVal URL As String, ByVal Path As SFile, ByVal AskForPath As Boolean) As IEnumerable(Of UserMedia)
If IsMyClass Then
Return DirectCast(Source, SiteSettingsBase).GetSpecialDataF(URL)
Else
Dim um As IEnumerable(Of IPluginUserMedia) = Source.GetSpecialData(URL)
Dim um As IEnumerable(Of PluginUserMedia) = Source.GetSpecialData(URL, Path, AskForPath)
If um.ListExists Then
Dim u As New List(Of UserMedia)
For Each d As IPluginUserMedia In um : u.Add(New UserMedia(d)) : Next
For Each d As PluginUserMedia In um : u.Add(New UserMedia(d)) : Next
Return u
End If
Return Nothing

View File

@@ -27,7 +27,7 @@ Namespace Plugin.Hosts
End Sub
Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)
If Loading Then
ExternalPlugin.XmlFieldsSet(Container.ToKeyValuePair)
ExternalPlugin.XmlFieldsSet(ToKeyValuePair(Of String, EContainer)(Container))
Else
Dim fl As List(Of KeyValuePair(Of String, String)) = ExternalPlugin.XmlFieldsGet
If fl.ListExists Then
@@ -53,11 +53,16 @@ Namespace Plugin.Hosts
.PostsNumberLimit = DownloadTopCount
.PostsDateLimit = DownloadToDate
.ExistingContentList = New List(Of PluginUserMedia)
.TempMediaList = New List(Of PluginUserMedia)
.TempPostsList = New List(Of String)
If _ContentList.Count > 0 Then ExternalPlugin.ExistingContentList = _ContentList.Select(Function(u) u.PluginUserMedia).ToList
ExternalPlugin.TempPostsList = ListAddList(Nothing, _TempPostsList)
.GetMedia()
_TempPostsList.ListAddList(.TempPostsList, LNC)
If .TempMediaList.ListExists Then _TempMediaList.ListAddList(.TempMediaList.Select(Function(tm) New UserMedia(tm)), LNC)
If Not .Name = Name Then Name = .Name
@@ -70,7 +75,18 @@ Namespace Plugin.Hosts
Protected Overrides Sub ReparseVideo(ByVal Token As CancellationToken)
End Sub
Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
If UseInternalDownloader Then DownloadContentDefault(Token) Else ExternalPlugin.Download()
If UseInternalDownloader Then
DownloadContentDefault(Token)
Else
With ExternalPlugin
If .TempMediaList.ListExists Then .TempMediaList.Clear()
.TempMediaList = New List(Of PluginUserMedia)
.TempMediaList.ListAddList(_ContentNew.Select(Function(c) c.PluginUserMedia()))
.Download()
_ContentNew.Clear()
If .TempMediaList.ListExists Then _ContentNew.ListAddList(.TempMediaList.Select(Function(c) New UserMedia(c)))
End With
End If
End Sub
Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False) As Integer
LogError(ex, Message)
@@ -83,5 +99,16 @@ Namespace Plugin.Hosts
Private Sub ExternalPlugin_TotalCountChanged(ByVal Count As Integer)
Progress.TotalCount += Count
End Sub
Protected Overrides Sub Dispose(ByVal disposing As Boolean)
If disposing And Not disposedValue Then
With ExternalPlugin
If .ExistingContentList.ListExists Then .ExistingContentList.Clear()
If .TempMediaList.ListExists Then .TempMediaList.Clear()
If .TempPostsList.ListExists Then .TempPostsList.Clear()
.Dispose()
End With
End If
MyBase.Dispose(disposing)
End Sub
End Class
End Namespace

View File

@@ -271,7 +271,6 @@
<DesignTimeSharedInput>True</DesignTimeSharedInput>
</Compile>
<Compile Include="PluginsEnvironment\Attributes\AttributesInternal.vb" />
<Compile Include="PluginsEnvironment\Hosts\IResponserContainer.vb" />
<Compile Include="PluginsEnvironment\Hosts\LogHost.vb" />
<Compile Include="PluginsEnvironment\Hosts\PropertyValueHost.vb" />
<Compile Include="PluginsEnvironment\Hosts\PluginHost.vb" />