mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-15 00:02:17 +00:00
2.0.0.0
Added Instagram downloading, filter by site, channels groups, change folder function, imgur compatibility, special folders, deleting with keeping data, Reddit saved posts downloading Fixed limited twitter downloading, suspended profiles Updated download algo Concat sites editors into a single form Updated Reddit downloading algo Fixed saved function in video downloader Some improvements
This commit is contained in:
Andy
13
SCrawler/API/Base/Declarations.vb
Normal file
13
SCrawler/API/Base/Declarations.vb
Normal file
@@ -0,0 +1,13 @@
|
||||
' Copyright (C) 2022 Andy
|
||||
' This program is free software: you can redistribute it and/or modify
|
||||
' it under the terms of the GNU General Public License as published by
|
||||
' the Free Software Foundation, either version 3 of the License, or
|
||||
' (at your option) any later version.
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Namespace API.Base
|
||||
Friend Module Declarations
|
||||
Friend ReadOnly LNC As New ListAddParams(LAP.NotContainsOnly)
|
||||
End Module
|
||||
End Namespace
|
||||
@@ -25,8 +25,47 @@ Namespace API.Base
|
||||
_Path.Value = NewFile
|
||||
End Set
|
||||
End Property
|
||||
Friend ReadOnly Property InstaHash As XMLValue(Of String)
|
||||
Friend ReadOnly Property InstaHashUpdateRequired As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property InstagramDownloadingErrorDate As XMLValue(Of Date)
|
||||
Friend Property InstagramLastApplyingValue As Integer? = Nothing
|
||||
Friend ReadOnly Property InstagramReadyForDownload As Boolean
|
||||
Get
|
||||
With InstagramDownloadingErrorDate
|
||||
If .ValueF.Exists Then
|
||||
Return .ValueF.Value.AddMinutes(If(InstagramLastApplyingValue, 10)) < Now
|
||||
Else
|
||||
Return True
|
||||
End If
|
||||
End With
|
||||
End Get
|
||||
End Property
|
||||
Friend Property InstagramTooManyRequestsReadyForCatch As Boolean = True
|
||||
Friend Sub InstagramTooManyRequests(ByVal Catched As Boolean)
|
||||
With InstagramDownloadingErrorDate
|
||||
If Catched Then
|
||||
If Not .ValueF.Exists Then
|
||||
.Value = Now
|
||||
If InstagramTooManyRequestsReadyForCatch Then
|
||||
InstagramLastApplyingValue = If(InstagramLastApplyingValue, 0) + 10
|
||||
InstagramTooManyRequestsReadyForCatch = False
|
||||
MyMainLOG = $"Instagram downloading error: too many requests. Try again after {If(InstagramLastApplyingValue, 10)} minutes..."
|
||||
End If
|
||||
End If
|
||||
Else
|
||||
.ValueF = Nothing
|
||||
InstagramLastApplyingValue = Nothing
|
||||
End If
|
||||
End With
|
||||
End Sub
|
||||
Friend ReadOnly Property Temporary As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property DownloadImages As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property DownloadVideos As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property GetUserMediaOnly As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property SavedPostsUserName As XMLValue(Of String)
|
||||
Private ReadOnly SettingsFile As SFile
|
||||
Friend Sub New(ByVal s As Sites, ByRef _XML As XmlFile, ByVal GlobalPath As SFile)
|
||||
Friend Sub New(ByVal s As Sites, ByRef _XML As XmlFile, ByVal GlobalPath As SFile,
|
||||
ByRef _Temp As XMLValue(Of Boolean), ByRef _Imgs As XMLValue(Of Boolean), ByRef _Vids As XMLValue(Of Boolean))
|
||||
Site = s
|
||||
SettingsFile = $"{SettingsFolderName}\Responser_{s}.xml"
|
||||
Responser = New WEB.Response(SettingsFile)
|
||||
@@ -34,40 +73,112 @@ Namespace API.Base
|
||||
If SettingsFile.Exists Then
|
||||
Responser.LoadSettings()
|
||||
Else
|
||||
If Site = Sites.Twitter Then
|
||||
With Responser
|
||||
.ContentType = "application/json"
|
||||
.Accept = "*/*"
|
||||
.CookiesDomain = "twitter.com"
|
||||
.Decoders.Add(SymbolsConverter.Converters.Unicode)
|
||||
With .Headers
|
||||
.Add("sec-ch-ua", " Not;A Brand" & Chr(34) & ";v=" & Chr(34) & "99" & Chr(34) & ", " & Chr(34) &
|
||||
"Google Chrome" & Chr(34) & ";v=" & Chr(34) & "91" & Chr(34) & ", " & Chr(34) & "Chromium" &
|
||||
Chr(34) & ";v=" & Chr(34) & "91" & Chr(34))
|
||||
.Add("sec-ch-ua-mobile", "?0")
|
||||
.Add("sec-fetch-dest", "empty")
|
||||
.Add("sec-fetch-mode", "cors")
|
||||
.Add("sec-fetch-site", "same-origin")
|
||||
.Add(Header_Twitter_Token, String.Empty)
|
||||
.Add("x-twitter-active-user", "yes")
|
||||
.Add("x-twitter-auth-type", "OAuth2Session")
|
||||
.Add(Header_Twitter_Authorization, String.Empty)
|
||||
Select Case Site
|
||||
Case Sites.Twitter
|
||||
With Responser
|
||||
.ContentType = "application/json"
|
||||
.Accept = "*/*"
|
||||
.CookiesDomain = "twitter.com"
|
||||
.Decoders.Add(SymbolsConverter.Converters.Unicode)
|
||||
With .Headers
|
||||
.Add("sec-ch-ua", " Not;A Brand" & Chr(34) & ";v=" & Chr(34) & "99" & Chr(34) & ", " & Chr(34) &
|
||||
"Google Chrome" & Chr(34) & ";v=" & Chr(34) & "91" & Chr(34) & ", " & Chr(34) & "Chromium" &
|
||||
Chr(34) & ";v=" & Chr(34) & "91" & Chr(34))
|
||||
.Add("sec-ch-ua-mobile", "?0")
|
||||
.Add("sec-fetch-dest", "empty")
|
||||
.Add("sec-fetch-mode", "cors")
|
||||
.Add("sec-fetch-site", "same-origin")
|
||||
.Add(Header_Twitter_Token, String.Empty)
|
||||
.Add("x-twitter-active-user", "yes")
|
||||
.Add("x-twitter-auth-type", "OAuth2Session")
|
||||
.Add(Header_Twitter_Authorization, String.Empty)
|
||||
End With
|
||||
End With
|
||||
End With
|
||||
ElseIf Site = Sites.Reddit Then
|
||||
Responser.CookiesDomain = "reddit.com"
|
||||
Responser.Decoders.Add(SymbolsConverter.Converters.Unicode)
|
||||
End If
|
||||
Case Sites.Reddit
|
||||
Responser.CookiesDomain = "reddit.com"
|
||||
Responser.Decoders.Add(SymbolsConverter.Converters.Unicode)
|
||||
Case Sites.Instagram : Responser.CookiesDomain = "instagram.com"
|
||||
End Select
|
||||
Responser.SaveSettings()
|
||||
End If
|
||||
_Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"),
|
||||
_XML, {SettingsCLS.Name_Node_Sites, Site.ToString}, XMLValue(Of SFile).ToFilePath)
|
||||
|
||||
Dim n() As String = {SettingsCLS.Name_Node_Sites, Site.ToString}
|
||||
_Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"), _XML, n, XMLValue(Of SFile).ToFilePath)
|
||||
_Path.ReplaceByValue("Path", {Site.ToString})
|
||||
_XML.Remove(Site.ToString)
|
||||
|
||||
Temporary = New XMLValue(Of Boolean)
|
||||
Temporary.SetExtended("Temporary", False, _XML, n)
|
||||
Temporary.SetDefault(_Temp)
|
||||
|
||||
DownloadImages = New XMLValue(Of Boolean)
|
||||
DownloadImages.SetExtended("DownloadImages", True, _XML, n)
|
||||
DownloadImages.SetDefault(_Imgs)
|
||||
|
||||
DownloadVideos = New XMLValue(Of Boolean)
|
||||
DownloadVideos.SetExtended("DownloadVideos", True, _XML, n)
|
||||
DownloadVideos.SetDefault(_Vids)
|
||||
|
||||
If Site = Sites.Twitter Then
|
||||
GetUserMediaOnly = New XMLValue(Of Boolean)("GetUserMediaOnly", True, _XML, n)
|
||||
GetUserMediaOnly.ReplaceByValue("TwitterDefaultGetUserMedia", n)
|
||||
Else
|
||||
GetUserMediaOnly = New XMLValue(Of Boolean)
|
||||
End If
|
||||
|
||||
If Site = Sites.Instagram Then
|
||||
InstaHash = New XMLValue(Of String)("InstaHash", String.Empty, _XML, n)
|
||||
InstaHashUpdateRequired = New XMLValue(Of Boolean)("InstaHashUpdateRequired", True, _XML, n)
|
||||
If (InstaHash.IsEmptyString Or InstaHashUpdateRequired) And Responser.Cookies.ListExists Then GatherInstaHash()
|
||||
InstagramDownloadingErrorDate = New XMLValue(Of Date) With {.ToStringFunction = Function(ss, vv) AConvert(Of String)(vv, Nothing)}
|
||||
InstagramDownloadingErrorDate.SetExtended("InstagramDownloadingErrorDate", Now.AddYears(-10), _XML, n)
|
||||
Else
|
||||
InstaHash = New XMLValue(Of String)
|
||||
InstaHashUpdateRequired = New XMLValue(Of Boolean)
|
||||
End If
|
||||
If Site = Sites.Reddit Then
|
||||
SavedPostsUserName = New XMLValue(Of String)("SavedPostsUserName", String.Empty, _XML, n)
|
||||
Else
|
||||
SavedPostsUserName = New XMLValue(Of String)
|
||||
End If
|
||||
End Sub
|
||||
Friend Sub Update()
|
||||
Responser.SaveSettings()
|
||||
End Sub
|
||||
Friend Function GatherInstaHash() As Boolean
|
||||
Try
|
||||
Dim rs As New RegexStructure("=" & Chr(34) & "([^" & Chr(34) & "]+?ConsumerLibCommons[^" & Chr(34) & "]+?.js)" & Chr(34), 1) With {
|
||||
.UseTimeOut = True,
|
||||
.MatchTimeOutSeconds = 10
|
||||
}
|
||||
Dim r$ = Responser.GetResponse("https://instagram.com",, EDP.ThrowException)
|
||||
If Not r.IsEmptyString Then
|
||||
Dim hStr$ = RegexReplace(r, rs)
|
||||
If Not hStr.IsEmptyString Then
|
||||
Do While Left(hStr, 1) = "/" : hStr = Right(hStr, hStr.Length - 1) : Loop
|
||||
hStr = $"https://instagram.com/{hStr}"
|
||||
r = Responser.GetResponse(hStr,, EDP.ThrowException)
|
||||
If Not r.IsEmptyString Then
|
||||
rs = New RegexStructure("generatePaginationActionCreators.+?.profilePosts.byUserId.get.+?queryId:.([\d\w\S]+?)" & Chr(34), 1) With {
|
||||
.UseTimeOut = True,
|
||||
.MatchTimeOutSeconds = 10
|
||||
}
|
||||
Dim h$ = RegexReplace(r, rs)
|
||||
If Not h.IsEmptyString Then
|
||||
InstaHash.Value = h
|
||||
InstaHashUpdateRequired.Value = False
|
||||
Return True
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
Return False
|
||||
Catch ex As Exception
|
||||
InstaHashUpdateRequired.Value = True
|
||||
InstaHash.Value = String.Empty
|
||||
Return ErrorsDescriber.Execute(EDP.SendInLog + EDP.ReturnValue, ex, "[SiteSettings.GaterInstaHash]", False)
|
||||
End Try
|
||||
End Function
|
||||
#Region "IDisposable Support"
|
||||
Private disposedValue As Boolean = False
|
||||
Protected Overridable Overloads Sub Dispose(ByVal disposing As Boolean)
|
||||
|
||||
@@ -78,6 +78,7 @@ Namespace API.Base
|
||||
#Region "Declarations"
|
||||
Friend MustOverride Property Site As Sites Implements IContentProvider.Site
|
||||
Friend User As UserInfo
|
||||
Friend Property IsSavedPosts As Boolean
|
||||
Protected Const NonExistendUserHelp As String = "404"
|
||||
Protected Const SuspendedUserHelp As String = "403"
|
||||
Friend Overridable Property UserExists As Boolean = True Implements IUserData.Exists
|
||||
@@ -329,17 +330,16 @@ BlockNullPicture:
|
||||
Dim luv$ = String.Empty
|
||||
If LastUpdated.HasValue Then luv = $"{LastUpdated.Value.ToStringDate(ADateTime.Formats.BaseDateTime)}: "
|
||||
Return $"{luv}{Name} [{Site}]{IIf(HasError, " (with errors)", String.Empty)}: P - {_DownloadedPicturesTotal}; V - {_DownloadedVideosTotal}" &
|
||||
$" (P - {_CountPictures}; V - {_CountVideo})"
|
||||
$" (P - {_CountPictures}; V - {_CountVideo})"
|
||||
End Get
|
||||
End Property
|
||||
#End Region
|
||||
Protected ReadOnly LNC As New ListAddParams(LAP.NotContainsOnly)
|
||||
#End Region
|
||||
#Region "LVI"
|
||||
Friend ReadOnly Property LVIKey As String Implements IUserData.LVIKey
|
||||
Get
|
||||
If Not _IsCollection Then
|
||||
Return $"{IIf(Site = Sites.Reddit, "R", "T")}_{Name}"
|
||||
Return $"{Interaction.Switch(Site = Sites.Reddit, "R", Site = Sites.Twitter, "T", Site = Sites.Instagram, "I")}_{Name}"
|
||||
Else
|
||||
Return $"CCCC_{CollectionName}"
|
||||
End If
|
||||
@@ -361,14 +361,18 @@ BlockNullPicture:
|
||||
End Function
|
||||
Friend Overridable ReadOnly Property FitToAddParams As Boolean Implements IUserData.FitToAddParams
|
||||
Get
|
||||
Select Case Settings.ShowingMode.Value
|
||||
Case ShowingModes.Regular : Return Not Temporary And Not Favorite
|
||||
Case ShowingModes.Temporary : Return Temporary
|
||||
Case ShowingModes.Favorite : Return Favorite
|
||||
Case ShowingModes.Labels : Return Settings.Labels.CurrentSelection.ListContains(Labels)
|
||||
Case ShowingModes.NoLabels : Return Labels.Count = 0
|
||||
Case Else : Return True
|
||||
End Select
|
||||
If Settings.SelectedSites.Count = 0 OrElse Settings.SelectedSites.Contains(Site) Then
|
||||
Select Case Settings.ShowingMode.Value
|
||||
Case ShowingModes.Regular : Return Not Temporary And Not Favorite
|
||||
Case ShowingModes.Temporary : Return Temporary
|
||||
Case ShowingModes.Favorite : Return Favorite
|
||||
Case ShowingModes.Labels : Return Settings.Labels.CurrentSelection.ListContains(Labels)
|
||||
Case ShowingModes.NoLabels : Return Labels.Count = 0
|
||||
Case Else : Return True
|
||||
End Select
|
||||
Else
|
||||
Return False
|
||||
End If
|
||||
End Get
|
||||
End Property
|
||||
Friend Function GetLVIGroup(ByVal Destination As ListView) As ListViewGroup Implements IUserData.GetLVIGroup
|
||||
@@ -383,7 +387,7 @@ BlockNullPicture:
|
||||
Return Destination.Groups.Item(LabelsKeeper.NoLabeledName)
|
||||
End If
|
||||
Else
|
||||
Return Destination.Groups.Item(GetLviGroupName(Site, Temporary, Favorite, IsCollection))
|
||||
Return Destination.Groups.Item(GetLviGroupName(Site, Temporary, Favorite, IsCollection, IsChannel))
|
||||
End If
|
||||
Catch ex As Exception
|
||||
Return Destination.Groups.Item(LabelsKeeper.NoLabeledName)
|
||||
@@ -431,6 +435,7 @@ BlockNullPicture:
|
||||
Return New Reddit.UserData(u, _LoadUserInformation)
|
||||
End If
|
||||
Case Sites.Twitter : Return New Twitter.UserData(u, _LoadUserInformation)
|
||||
Case Sites.Instagram : Return New Instagram.UserData(u, _LoadUserInformation)
|
||||
Case Else : Throw New ArgumentOutOfRangeException("Site", $"Site [{u.Site}] information does not recognized by loader")
|
||||
End Select
|
||||
End Function
|
||||
@@ -503,7 +508,7 @@ BlockNullPicture:
|
||||
|
||||
x.Save(MyFile)
|
||||
End Using
|
||||
Settings.UpdateUsersList(User)
|
||||
If Not IsSavedPosts Then Settings.UpdateUsersList(User)
|
||||
Catch ex As Exception
|
||||
LogError(ex, "user information saving error")
|
||||
End Try
|
||||
@@ -584,7 +589,8 @@ BlockNullPicture:
|
||||
Select Case Site
|
||||
Case Sites.Reddit : URL = $"https://www.reddit.com/{IIf(IsChannel, "r", "user")}/{Name}/"
|
||||
Case Sites.Twitter : URL = $"https://twitter.com/{Name}"
|
||||
Case Else : MsgBoxE($"Site [{Site}] opening does not implemented", MsgBoxStyle.Exclamation)
|
||||
Case Sites.Instagram : URL = $"https://www.instagram.com/{Name}/"
|
||||
Case Else : MsgBoxE($"Site [{Site}] opening not implemented", MsgBoxStyle.Exclamation)
|
||||
End Select
|
||||
If Not URL.IsEmptyString Then Process.Start(URL)
|
||||
Catch ex As Exception
|
||||
@@ -604,10 +610,12 @@ BlockNullPicture:
|
||||
UpdateDataFiles()
|
||||
If Not Responser Is Nothing Then Responser.Dispose()
|
||||
Responser = New PersonalUtilities.Tools.WEB.Response
|
||||
Responser.Copy(Settings.Site(Site).Responser)
|
||||
Responser.Copy(Settings(Site).Responser)
|
||||
Dim UpPic As Boolean = Settings.ViewModeIsPicture AndAlso GetPicture(False) Is Nothing
|
||||
Dim sEnvir() As Boolean = {UserExists, UserSuspended}
|
||||
Dim EnvirChanged As Func(Of Boolean) = Function() Not sEnvir(0) = UserExists Or Not sEnvir(1) = UserSuspended
|
||||
UserExists = True
|
||||
UserSuspended = False
|
||||
_DownloadedPicturesSession = 0
|
||||
_DownloadedVideosSession = 0
|
||||
_TempMediaList.Clear()
|
||||
@@ -714,7 +722,7 @@ BlockNullPicture:
|
||||
End Function
|
||||
Friend Function DeleteF(ByVal Instance As IUserData) As Integer
|
||||
Dim f As SFile = SFile.GetPath(MyFile.CutPath.Path)
|
||||
If f.Exists(SFO.Path, False) AndAlso f.Delete(SFO.Path, False, False) Then
|
||||
If f.Exists(SFO.Path, False) AndAlso (User.Merged OrElse f.Delete(SFO.Path, False, False)) Then
|
||||
ImageHandler(Me, False)
|
||||
Settings.UsersList.Remove(User)
|
||||
Settings.UpdateUsersList()
|
||||
@@ -919,7 +927,7 @@ BlockNullPicture:
|
||||
#End Region
|
||||
#Region "IEquatable Support"
|
||||
Friend Overridable Overloads Function Equals(ByVal Other As UserDataBase) As Boolean Implements IEquatable(Of UserDataBase).Equals
|
||||
Return Site = Other.Site And Name = Other.Name
|
||||
Return Site = Other.Site And Name = Other.Name And IsSavedPosts = Other.IsSavedPosts
|
||||
End Function
|
||||
Public Overrides Function Equals(ByVal Obj As Object) As Boolean
|
||||
If TypeOf Obj Is Reddit.Channel Then
|
||||
|
||||
Reference in New Issue
Block a user