mirror of
https://github.com/AAndyProgram/SCrawler.git
synced 2026-03-15 00:02:17 +00:00
2023.6.5.0
YT settings: removed property 'ItemsListLimit', add property 'ReplaceModificationDate' YT.MediaItem: fix 'Pending' YT.VideoListForm: add 'Shift' to add without downloading; add 'F5' hot key to start download; remove list items limit; fix item 'Pending', fixed items queue UserDataBase: add 'IconBannerDownloaded' properties; add 'HOST.Available' check to 'DownloadSingleObject'; update file deletion in 'DownloadContentDefault'; add truncating '_TempPostsList' if number of ids > 1000 Instagram: add authorization headers Mastodon: implement 'DownloadIconBanner'; update 'ReparseMissing' function Reddit: implement 'DownloadIconBanner' Twitter: implement 'DownloadIconBanner'; update parsers to parse posts with two videos; implement gallery-dl for all function; remove headers from settings Download.DownloadProgress: remove main progress perform when downloading saved posts VideoDownloaderForm: bind the 'BTT_ADD_URLS_ARR' button to the 'BTT_ADD_KeyClick' function UsersInfoForm: add folder opening on double click on an item ListImagesLoader: fix refill bug when the number of filtered profiles = 0 TrayIcon: add standalone downloader to context menu DownloadableMediaHost: fix a bug when not downloaded videos do not appear in the list when loading the program
This commit is contained in:
Andy
@@ -72,15 +72,22 @@ Namespace API.Base.GDL
|
||||
Friend Const UrlTextStart As String = UrlLibStart & " https"
|
||||
Friend Sub New()
|
||||
MyBase.New(True)
|
||||
MainProcessName = "gallery-dl"
|
||||
ChangeDirectory(Settings.GalleryDLFile.File)
|
||||
End Sub
|
||||
Public Overrides Sub Create()
|
||||
If TempPostsList Is Nothing Then TempPostsList = New List(Of String)
|
||||
MyBase.Create()
|
||||
End Sub
|
||||
Protected Overrides Async Sub OutputDataReceiver(ByVal Sender As Object, ByVal e As DataReceivedEventArgs)
|
||||
MyBase.OutputDataReceiver(Sender, e)
|
||||
Await Validate(e.Data)
|
||||
If Not ProcessKilled Then
|
||||
MyBase.OutputDataReceiver(Sender, e)
|
||||
Await Validate(e.Data)
|
||||
End If
|
||||
End Sub
|
||||
Protected Overridable Async Function Validate(ByVal Value As String) As Task
|
||||
If Await Task.Run(Of Boolean)(Function() Not Value.IsEmptyString AndAlso
|
||||
TempPostsList.Exists(Function(v) Value.Contains(v))) Then Kill(EDP.None)
|
||||
If Not ProcessKilled AndAlso Await Task.Run(Of Boolean)(Function() Not Value.IsEmptyString AndAlso
|
||||
TempPostsList.Exists(Function(v) Value.Contains(v))) Then Kill()
|
||||
End Function
|
||||
End Class
|
||||
End Namespace
|
||||
@@ -126,6 +126,7 @@ Namespace API.Base
|
||||
Private Const Name_ReadyForDownload As String = "ReadyForDownload"
|
||||
Private Const Name_DownloadImages As String = "DownloadImages"
|
||||
Private Const Name_DownloadVideos As String = "DownloadVideos"
|
||||
Private Const Name_IconBannerDownloaded As String = "IconBannerDownloaded"
|
||||
|
||||
Private Const Name_VideoCount As String = "VideoCount"
|
||||
Private Const Name_PicturesCount As String = "PicturesCount"
|
||||
@@ -434,6 +435,18 @@ BlockNullPicture:
|
||||
Friend Property DownloadImages As Boolean = True Implements IUserData.DownloadImages
|
||||
Friend Property DownloadVideos As Boolean = True Implements IUserData.DownloadVideos
|
||||
Friend Property DownloadMissingOnly As Boolean = False Implements IUserData.DownloadMissingOnly
|
||||
Private _IconBannerDownloaded As Boolean = False
|
||||
Friend WriteOnly Property IconBannerDownloaded As Boolean
|
||||
Set(ByVal IsDownloaded As Boolean)
|
||||
If Not _IconBannerDownloaded = IsDownloaded Then _ForceSaveUserInfo = True
|
||||
_IconBannerDownloaded = IsDownloaded
|
||||
End Set
|
||||
End Property
|
||||
Friend ReadOnly Property DownloadIconBanner As Boolean
|
||||
Get
|
||||
Return Not _IconBannerDownloaded Or Settings.UpdateUserIconBannerEveryTime
|
||||
End Get
|
||||
End Property
|
||||
#End Region
|
||||
#Region "Content"
|
||||
Protected ReadOnly _ContentList As List(Of UserMedia)
|
||||
@@ -751,6 +764,7 @@ BlockNullPicture:
|
||||
ReadyForDownload = x.Value(Name_ReadyForDownload).FromXML(Of Boolean)(True)
|
||||
DownloadImages = x.Value(Name_DownloadImages).FromXML(Of Boolean)(True)
|
||||
DownloadVideos = x.Value(Name_DownloadVideos).FromXML(Of Boolean)(True)
|
||||
_IconBannerDownloaded = x.Value(Name_IconBannerDownloaded).FromXML(Of Boolean)(False)
|
||||
DownloadedVideos(True) = x.Value(Name_VideoCount).FromXML(Of Integer)(0)
|
||||
DownloadedPictures(True) = x.Value(Name_PicturesCount).FromXML(Of Integer)(0)
|
||||
LastUpdated = AConvert(Of Date)(x.Value(Name_LastUpdated), ADateTime.Formats.BaseDateTime, Nothing)
|
||||
@@ -799,6 +813,7 @@ BlockNullPicture:
|
||||
x.Add(Name_ReadyForDownload, ReadyForDownload.BoolToInteger)
|
||||
x.Add(Name_DownloadImages, DownloadImages.BoolToInteger)
|
||||
x.Add(Name_DownloadVideos, DownloadVideos.BoolToInteger)
|
||||
x.Add(Name_IconBannerDownloaded, _IconBannerDownloaded.BoolToInteger)
|
||||
x.Add(Name_VideoCount, DownloadedVideos(True))
|
||||
x.Add(Name_PicturesCount, DownloadedPictures(True))
|
||||
x.Add(Name_LastUpdated, AConvert(Of String)(LastUpdated, ADateTime.Formats.BaseDateTime, String.Empty))
|
||||
@@ -934,12 +949,12 @@ BlockNullPicture:
|
||||
_EnvirUserExists = UserExists
|
||||
_EnvirUserSuspended = UserSuspended
|
||||
_EnvirChanged = False
|
||||
_EnvirInvokeUserUpdated = False
|
||||
UserExists = True
|
||||
UserSuspended = False
|
||||
DownloadedPictures(False) = 0
|
||||
DownloadedVideos(False) = 0
|
||||
_PictureExists = Settings.ViewModeIsPicture AndAlso Not GetPicture(Of Image)(False) Is Nothing
|
||||
_EnvirInvokeUserUpdated = False
|
||||
End Sub
|
||||
Private Sub EnvirChanged(ByVal NewValue As Object, <CallerMemberName> Optional ByVal Caller As String = Nothing)
|
||||
If _DownloadInProgress Then
|
||||
@@ -997,8 +1012,10 @@ BlockNullPicture:
|
||||
|
||||
If UseMD5Comparison Then ValidateMD5(Token) : ProgressPre.Done() : ThrowAny(Token)
|
||||
|
||||
If _TempPostsList.Count > 0 And Not DownloadMissingOnly And __SaveData Then _
|
||||
TextSaver.SaveTextToFile(_TempPostsList.ListToString(Environment.NewLine), MyFilePosts, True,, EDP.None)
|
||||
If _TempPostsList.Count > 0 And Not DownloadMissingOnly And __SaveData Then
|
||||
If _TempPostsList.Count > 1000 Then _TempPostsList.ListAddList(_TempPostsList.ListTake(-2, 1000, EDP.ReturnValue).ListReverse, LAP.ClearBeforeAdd)
|
||||
TextSaver.SaveTextToFile(_TempPostsList.ListToString(Environment.NewLine), MyFilePosts, True,, EDP.None)
|
||||
End If
|
||||
_ContentNew.ListAddList(_TempMediaList, LAP.ClearBeforeAdd)
|
||||
DownloadContent(Token)
|
||||
ThrowIfDisposed()
|
||||
@@ -1079,7 +1096,8 @@ BlockNullPicture:
|
||||
Progress = Data.Progress
|
||||
If Not Responser Is Nothing Then Responser.Dispose()
|
||||
Responser = New Responser
|
||||
If Not HOST Is Nothing AndAlso Not HOST.Responser Is Nothing Then Responser.Copy(HOST.Responser)
|
||||
If Not HOST Is Nothing AndAlso HOST.Available(ISiteSettings.Download.SingleObject, True) AndAlso
|
||||
Not HOST.Responser Is Nothing Then Responser.Copy(HOST.Responser)
|
||||
SeparateVideoFolder = False
|
||||
IsSingleObjectDownload = True
|
||||
DownloadSingleObject_GetPosts(Data, Token)
|
||||
@@ -1338,6 +1356,16 @@ BlockNullPicture:
|
||||
Dim f As SFile
|
||||
Dim v As UserMedia
|
||||
Dim fileNumProvider As SFileNumbers = SFileNumbers.Default
|
||||
Dim __deleteFile As Action(Of SFile, String) = Sub(ByVal FileToDelete As SFile, ByVal FileUrl As String)
|
||||
Try
|
||||
If FileToDelete.Exists Then FileToDelete.Delete(,, EDP.ThrowException)
|
||||
Catch file_io_ex As IOException
|
||||
MyMainLOG = "File download aborted. You should download the following file again." & vbCr &
|
||||
$"File: {FileToDelete}{vbCr}URL: {FileUrl}"
|
||||
Catch file_del_ex As Exception
|
||||
ErrorsDescriber.Execute(EDP.SendToLog, file_del_ex)
|
||||
End Try
|
||||
End Sub
|
||||
|
||||
Using w As New OptionalWebClient(Me)
|
||||
If vsf Then CSFileP($"{MyDir}\Video\").Exists(SFO.Path)
|
||||
@@ -1426,7 +1454,9 @@ BlockNullPicture:
|
||||
DownloadContentDefault_PostProcessing(v, f, Token)
|
||||
dCount += 1
|
||||
Catch woex As OperationCanceledException When Token.IsCancellationRequested
|
||||
If f.Exists Then f.Delete()
|
||||
'TODELETE: UserDataBase.DownloadContentDefault: remove file when 'OperationCanceledException'
|
||||
'If f.Exists Then f.Delete(,, EDP.SendToLog)
|
||||
__deleteFile.Invoke(f, v.URL_BASE)
|
||||
v.State = UStates.Missing
|
||||
v.Attempts += 1
|
||||
_ContentNew(i) = v
|
||||
|
||||
@@ -46,6 +46,7 @@ Namespace API.Instagram
|
||||
If Not token.IsEmptyString Then Destination.Headers.Add(SiteSettings.Header_CSRF_TOKEN, token)
|
||||
If Not isInternal Then
|
||||
Destination.Cookies.Update(Source.Cookies, CookieKeeper.UpdateModes.ReplaceByNameAll, False, EDP.SendToLog)
|
||||
Destination.Cookies.Update(EDP.SendToLog)
|
||||
Destination.SaveSettings()
|
||||
End If
|
||||
End If
|
||||
|
||||
@@ -70,33 +70,54 @@ Namespace API.Instagram
|
||||
End Class
|
||||
#End Region
|
||||
#Region "Authorization properties"
|
||||
<PropertyOption(ControlText:="Hash", ControlToolTip:="Instagram session hash for tagged posts", IsAuth:=True), PXML("InstaHash"), ControlNumber(0)>
|
||||
Friend ReadOnly Property HashTagged As PropertyValue
|
||||
<PropertyOption(ControlText:="x-csrftoken", IsAuth:=True, AllowNull:=False), ControlNumber(2)>
|
||||
Friend ReadOnly Property CSRF_TOKEN As PropertyValue
|
||||
<PropertyOption(ControlText:="x-ig-app-id", IsAuth:=True, AllowNull:=False), ControlNumber(3)>
|
||||
Friend Property IG_APP_ID As PropertyValue
|
||||
<PropertyOption(ControlText:="x-ig-www-claim", IsAuth:=True, AllowNull:=True), ControlNumber(4)>
|
||||
Friend Property IG_WWW_CLAIM As PropertyValue
|
||||
Friend Overrides Function BaseAuthExists() As Boolean
|
||||
Return Responser.CookiesExists And ACheck(IG_APP_ID.Value) And ACheck(CSRF_TOKEN.Value)
|
||||
End Function
|
||||
Private Const Header_IG_APP_ID As String = "x-ig-app-id"
|
||||
Friend Const Header_IG_WWW_CLAIM As String = "x-ig-www-claim"
|
||||
Friend Const Header_CSRF_TOKEN As String = "x-csrftoken"
|
||||
Private Const Header_ASBD_ID As String = "X-Asbd-Id"
|
||||
Private ReadOnly Header_Browser As New HttpHeader("Sec-Ch-Ua", """Google Chrome"";v=""113"", ""Chromium"";v=""113"", ""Not-A.Brand"";v=""24""")
|
||||
Private ReadOnly Header_BrowserExt As New HttpHeader("Sec-Ch-Ua-Full-Version-List", """Google Chrome"";v=""113.0.5672.127"", ""Chromium"";v=""113.0.5672.127"", ""Not-A.Brand"";v=""24.0.0.0""")
|
||||
Private ReadOnly Header_Platform As New HttpHeader("Sec-Ch-Ua-Platform-Version", """10.0.0""")
|
||||
<PropertyOption(ControlText:="Hash", ControlToolTip:="Instagram session hash for tagged posts", IsAuth:=True), PXML("InstaHash"), ControlNumber(0)>
|
||||
Friend ReadOnly Property HashTagged As PropertyValue
|
||||
<PropertyOption(ControlText:="x-csrftoken", IsAuth:=True, AllowNull:=False), ControlNumber(2)>
|
||||
Friend ReadOnly Property HH_CSRF_TOKEN As PropertyValue
|
||||
<PropertyOption(ControlText:="x-ig-app-id", IsAuth:=True, AllowNull:=False), ControlNumber(3)>
|
||||
Friend Property HH_IG_APP_ID As PropertyValue
|
||||
<PropertyOption(ControlText:="x-asbd-id", IsAuth:=True, AllowNull:=True), ControlNumber(4)>
|
||||
Friend Property HH_ASBD_ID As PropertyValue
|
||||
<PropertyOption(ControlText:="x-ig-www-claim", IsAuth:=True, AllowNull:=True), ControlNumber(5)>
|
||||
Friend Property HH_IG_WWW_CLAIM As PropertyValue
|
||||
<PropertyOption(ControlText:="sec-ch-ua", IsAuth:=True, AllowNull:=True), ControlNumber(6)>
|
||||
Private Property HH_BROWSER As PropertyValue
|
||||
<PropertyOption(ControlText:="sec-ch-ua-full", ControlToolTip:="sec-ch-ua-full-version-list", IsAuth:=True, AllowNull:=True), ControlNumber(7)>
|
||||
Private Property HH_BROWSER_EXT As PropertyValue
|
||||
<PropertyOption(ControlText:="sec-ch-ua-platform-ver", ControlToolTip:="sec-ch-ua-platform-version", IsAuth:=True, AllowNull:=True), ControlNumber(8)>
|
||||
Private Property HH_PLATFORM As PropertyValue
|
||||
<PropertyOption(ControlText:="UserAgent", IsAuth:=True, AllowNull:=True), ControlNumber(9)>
|
||||
Private Property HH_USER_AGENT As PropertyValue
|
||||
Friend Overrides Function BaseAuthExists() As Boolean
|
||||
Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value) And ACheck(HH_CSRF_TOKEN.Value)
|
||||
End Function
|
||||
Private _FieldsChangerSuspended As Boolean = False
|
||||
Private Sub ChangeResponserFields(ByVal PropName As String, ByVal Value As Object)
|
||||
If Not _FieldsChangerSuspended And Not PropName.IsEmptyString Then
|
||||
Dim f$ = String.Empty
|
||||
Dim isUserAgent As Boolean = False
|
||||
Select Case PropName
|
||||
Case NameOf(IG_APP_ID) : f = Header_IG_APP_ID
|
||||
Case NameOf(IG_WWW_CLAIM) : f = Header_IG_WWW_CLAIM
|
||||
Case NameOf(CSRF_TOKEN) : f = Header_CSRF_TOKEN
|
||||
Case NameOf(HH_IG_APP_ID) : f = Header_IG_APP_ID
|
||||
Case NameOf(HH_ASBD_ID) : f = Header_ASBD_ID
|
||||
Case NameOf(HH_IG_WWW_CLAIM) : f = Header_IG_WWW_CLAIM
|
||||
Case NameOf(HH_CSRF_TOKEN) : f = Header_CSRF_TOKEN
|
||||
Case NameOf(HH_BROWSER) : f = Header_Browser.Name
|
||||
Case NameOf(HH_BROWSER_EXT) : f = Header_BrowserExt.Name
|
||||
Case NameOf(HH_PLATFORM) : f = Header_Platform.Name
|
||||
Case NameOf(HH_USER_AGENT) : isUserAgent = True
|
||||
End Select
|
||||
If Not f.IsEmptyString Then
|
||||
Responser.Headers.Remove(f)
|
||||
If Not CStr(Value).IsEmptyString Then Responser.Headers.Add(f, CStr(Value))
|
||||
Responser.SaveSettings()
|
||||
ElseIf isUserAgent Then
|
||||
Responser.UserAgent = CStr(Value)
|
||||
End If
|
||||
End If
|
||||
End Sub
|
||||
@@ -192,13 +213,53 @@ Namespace API.Instagram
|
||||
Dim app_id$ = String.Empty
|
||||
Dim www_claim$ = String.Empty
|
||||
Dim token$ = String.Empty
|
||||
Dim asbd$ = String.Empty
|
||||
Dim browser$ = String.Empty
|
||||
Dim browserExt$ = String.Empty
|
||||
Dim platform$ = String.Empty
|
||||
Dim useragent$ = String.Empty
|
||||
|
||||
Dim __UpdateHeader As Action(Of HttpHeader, Boolean) = Sub(ByVal h As HttpHeader, ByVal UpdateValueIfEmpty As Boolean)
|
||||
With Responser.Headers
|
||||
Dim i% = .IndexOf(h)
|
||||
Dim hh As HttpHeader
|
||||
If i >= 0 Then
|
||||
hh = .Item(i)
|
||||
If hh.Value.IsEmptyString And UpdateValueIfEmpty Then hh.Value = h.Value
|
||||
Else
|
||||
hh = h
|
||||
End If
|
||||
.Add(hh)
|
||||
End With
|
||||
End Sub
|
||||
|
||||
With Responser
|
||||
If .Headers.Count > 0 Then
|
||||
token = .Headers.Value(Header_CSRF_TOKEN)
|
||||
app_id = .Headers.Value(Header_IG_APP_ID)
|
||||
www_claim = .Headers.Value(Header_IG_WWW_CLAIM)
|
||||
End If
|
||||
.Accept = "*/*"
|
||||
useragent = .UserAgent
|
||||
With .Headers
|
||||
If .Count > 0 Then
|
||||
token = .Value(Header_CSRF_TOKEN)
|
||||
app_id = .Value(Header_IG_APP_ID)
|
||||
www_claim = .Value(Header_IG_WWW_CLAIM)
|
||||
asbd = .Value(Header_ASBD_ID)
|
||||
browser = .Value(Header_Browser.Name)
|
||||
browserExt = .Value(Header_BrowserExt.Name)
|
||||
platform = .Value(Header_Platform.Name)
|
||||
End If
|
||||
.Add("Dnt", 1)
|
||||
__UpdateHeader(Header_Browser, browser.IsEmptyString)
|
||||
browser = .Value(Header_Browser.Name)
|
||||
__UpdateHeader(Header_BrowserExt, browserExt.IsEmptyString)
|
||||
browserExt = .Value(Header_BrowserExt.Name)
|
||||
.Add("Sec-Ch-Ua-Mobile", "?0")
|
||||
.Add("Sec-Ch-Ua-Platform", """Windows""")
|
||||
__UpdateHeader(Header_Platform, platform.IsEmptyString)
|
||||
platform = .Value(Header_Platform.Name)
|
||||
.Add("Sec-Fetch-Dest", "empty")
|
||||
.Add("Sec-Fetch-Mode", "cors")
|
||||
.Add("Sec-Fetch-Site", "same-origin")
|
||||
.Add("X-Requested-With", "XMLHttpRequest")
|
||||
End With
|
||||
.CookiesExtractMode = Responser.CookiesExtractModes.Response
|
||||
.CookiesUpdateMode = CookieKeeper.UpdateModes.ReplaceByNameAll
|
||||
.CookiesExtractedAutoSave = False
|
||||
@@ -207,9 +268,14 @@ Namespace API.Instagram
|
||||
Dim n() As String = {SettingsCLS.Name_Node_Sites, Site.ToString}
|
||||
|
||||
HashTagged = New PropertyValue(String.Empty, GetType(String))
|
||||
CSRF_TOKEN = New PropertyValue(token, GetType(String), Sub(v) ChangeResponserFields(NameOf(CSRF_TOKEN), v))
|
||||
IG_APP_ID = New PropertyValue(app_id, GetType(String), Sub(v) ChangeResponserFields(NameOf(IG_APP_ID), v))
|
||||
IG_WWW_CLAIM = New PropertyValue(www_claim.IfNullOrEmpty(0), GetType(String), Sub(v) ChangeResponserFields(NameOf(IG_WWW_CLAIM), v))
|
||||
HH_CSRF_TOKEN = New PropertyValue(token, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_CSRF_TOKEN), v))
|
||||
HH_IG_APP_ID = New PropertyValue(app_id, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_IG_APP_ID), v))
|
||||
HH_ASBD_ID = New PropertyValue(asbd, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_ASBD_ID), v))
|
||||
HH_IG_WWW_CLAIM = New PropertyValue(www_claim.IfNullOrEmpty(0), GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_IG_WWW_CLAIM), v))
|
||||
HH_BROWSER = New PropertyValue(browser, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_BROWSER), v))
|
||||
HH_BROWSER_EXT = New PropertyValue(browserExt, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_BROWSER_EXT), v))
|
||||
HH_PLATFORM = New PropertyValue(platform, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_PLATFORM), v))
|
||||
HH_USER_AGENT = New PropertyValue(useragent, GetType(String), Sub(v) ChangeResponserFields(NameOf(HH_USER_AGENT), v))
|
||||
|
||||
DownloadTimeline = New PropertyValue(True)
|
||||
DownloadStories = New PropertyValue(True)
|
||||
@@ -275,7 +341,7 @@ Namespace API.Instagram
|
||||
Private _NextTagged As Boolean = True
|
||||
Friend Overrides Sub DownloadStarted(ByVal What As Download)
|
||||
ActiveJobs += 1
|
||||
If LastDownloadDate.Value.AddMinutes(120) < Now Or Not ACheck(IG_WWW_CLAIM.Value) Then IG_WWW_CLAIM.Value = "0"
|
||||
If LastDownloadDate.Value.AddMinutes(120) < Now Or Not ACheck(HH_IG_WWW_CLAIM.Value) Then HH_IG_WWW_CLAIM.Value = "0"
|
||||
End Sub
|
||||
Friend Overrides Sub BeforeStartDownload(ByVal User As Object, ByVal What As Download)
|
||||
With DirectCast(User, UserData)
|
||||
@@ -299,8 +365,8 @@ Namespace API.Instagram
|
||||
LastDownloadDate.Value = Now
|
||||
LastRequestsCount.Value = .RequestsCount
|
||||
_FieldsChangerSuspended = True
|
||||
IG_WWW_CLAIM.Value = Responser.Headers.Value(Header_IG_WWW_CLAIM)
|
||||
CSRF_TOKEN.Value = Responser.Headers.Value(Header_CSRF_TOKEN)
|
||||
HH_IG_WWW_CLAIM.Value = Responser.Headers.Value(Header_IG_WWW_CLAIM)
|
||||
HH_CSRF_TOKEN.Value = Responser.Headers.Value(Header_CSRF_TOKEN)
|
||||
_FieldsChangerSuspended = False
|
||||
End With
|
||||
End Sub
|
||||
|
||||
@@ -13,6 +13,7 @@ Imports PersonalUtilities.Functions.XML
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Imports PersonalUtilities.Tools.Web.Documents.JSON
|
||||
Imports UTypes = SCrawler.API.Base.UserMedia.Types
|
||||
Imports UStates = SCrawler.API.Base.UserMedia.States
|
||||
Namespace API.Mastodon
|
||||
Friend Class UserData : Inherits Twitter.UserData
|
||||
#Region "XML names"
|
||||
@@ -136,11 +137,14 @@ Namespace API.Mastodon
|
||||
If __imgFile.Extension.IsEmptyString Then __imgFile.Extension = "jpg"
|
||||
__imgFile.Path = MyFile.CutPath.Path
|
||||
If Not __imgFile.Exists Then GetWebFile(img, __imgFile, EDP.None)
|
||||
If __imgFile.Exists Then IconBannerDownloaded = True
|
||||
End If
|
||||
End If
|
||||
End Sub
|
||||
__getImage.Invoke(.Value("header").IfNullOrEmpty(.Value("header_static")))
|
||||
__getImage.Invoke(.Value("avatar").IfNullOrEmpty(.Value("avatar_static")))
|
||||
If DownloadIconBanner Then
|
||||
__getImage.Invoke(.Value("header").IfNullOrEmpty(.Value("header_static")))
|
||||
__getImage.Invoke(.Value("avatar").IfNullOrEmpty(.Value("avatar_static")))
|
||||
End If
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
@@ -235,8 +239,44 @@ Namespace API.Mastodon
|
||||
Return $"https://{Domain}/api/v1/statuses/" & "{0}"
|
||||
End Function
|
||||
Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken)
|
||||
SinglePostUrl = GetSinglePostPattern(MyCredentials.Domain)
|
||||
MyBase.ReparseMissing(Token)
|
||||
Dim rList As New List(Of Integer)
|
||||
Dim URL$ = String.Empty
|
||||
Try
|
||||
If ContentMissingExists Then
|
||||
Dim SinglePostUrl$ = GetSinglePostPattern(MyCredentials.Domain)
|
||||
Dim m As UserMedia
|
||||
Dim r$, PostDate$
|
||||
Dim j As EContainer
|
||||
ProgressPre.ChangeMax(_ContentList.Count)
|
||||
For i% = 0 To _ContentList.Count - 1
|
||||
ProgressPre.Perform()
|
||||
If _ContentList(i).State = UStates.Missing Then
|
||||
m = _ContentList(i)
|
||||
If Not m.Post.ID.IsEmptyString Then
|
||||
ThrowAny(Token)
|
||||
URL = String.Format(SinglePostUrl, m.Post.ID)
|
||||
r = Responser.GetResponse(URL,, EDP.ReturnValue)
|
||||
If Not r.IsEmptyString Then
|
||||
j = JsonDocument.Parse(r)
|
||||
If Not j Is Nothing Then
|
||||
PostDate = String.Empty
|
||||
If j.Contains("created_at") Then PostDate = j("created_at").Value Else PostDate = String.Empty
|
||||
ObtainMedia(j, m.Post.ID, PostDate, UStates.Missing)
|
||||
rList.Add(i)
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, $"ReparseMissing error [{URL}]")
|
||||
Finally
|
||||
If rList.Count > 0 Then
|
||||
For i% = rList.Count - 1 To 0 Step -1 : _ContentList.RemoveAt(i) : Next
|
||||
rList.Clear()
|
||||
End If
|
||||
End Try
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "DownloadSingleObject"
|
||||
|
||||
@@ -270,11 +270,11 @@ Namespace API.Pinterest
|
||||
If IsBoardsRequested Then
|
||||
If ErrorOutputData.Count > 0 Then
|
||||
If Await Task.Run(Of Boolean)(Function() ErrorOutputData.Exists(Function(ee) Not ee.IsEmptyString AndAlso
|
||||
ee.StartsWith(UrlTextStart))) Then Kill(EDP.None)
|
||||
ee.StartsWith(UrlTextStart))) Then Kill()
|
||||
End If
|
||||
Else
|
||||
If Await Task.Run(Of Boolean)(Function() Not Value.IsEmptyString AndAlso
|
||||
Source._TempPostsList.Exists(Function(v) Value.Contains(v))) Then Kill(EDP.None)
|
||||
Source._TempPostsList.Exists(Function(v) Value.Contains(v))) Then Kill()
|
||||
End If
|
||||
End Function
|
||||
End Class
|
||||
|
||||
@@ -427,11 +427,14 @@ Namespace API.Reddit
|
||||
If f.Extension.IsEmptyString Then f.Extension = "jpg"
|
||||
f.Path = dir.Path
|
||||
If Not f.Exists Then GetWebFile(img, f, EDP.ReturnValue)
|
||||
If f.Exists Then IconBannerDownloaded = True
|
||||
End If
|
||||
End If
|
||||
End Sub
|
||||
__getFile.Invoke(.Value("icon_img"))
|
||||
__getFile.Invoke(.Value("banner_img"))
|
||||
If DownloadIconBanner Then
|
||||
__getFile.Invoke(.Value("icon_img"))
|
||||
__getFile.Invoke(.Value("banner_img"))
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
End Using
|
||||
|
||||
@@ -14,7 +14,6 @@ Namespace API.Twitter
|
||||
Friend Const TwitterSite As String = "Twitter"
|
||||
Friend Const TwitterSiteKey As String = "AndyProgram_Twitter"
|
||||
Friend ReadOnly DateProvider As ADateTime = GetDateProvider()
|
||||
Friend ReadOnly VideoNode As NodeParams() = {New NodeParams("video_info", True, True, True, True, 10)}
|
||||
Friend ReadOnly VideoSizeRegEx As RParams = RParams.DMS("\d+x(\d+)", 1, EDP.ReturnValue)
|
||||
Private Function GetDateProvider() As ADateTime
|
||||
Dim n As DateTimeFormatInfo = CultureInfo.GetCultureInfo("en-us").DateTimeFormat.Clone
|
||||
|
||||
@@ -12,7 +12,7 @@ Imports SCrawler.Plugin.Attributes
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Imports PersonalUtilities.Tools.Web.Clients
|
||||
Namespace API.Twitter
|
||||
<Manifest(TwitterSiteKey), SavedPosts, SpecialForm(False)>
|
||||
<Manifest(TwitterSiteKey), SavedPosts, SeparatedTasks, SpecialForm(False)>
|
||||
Friend Class SiteSettings : Inherits SiteSettingsBase
|
||||
#Region "Token names"
|
||||
Friend Const Header_Authorization As String = "authorization"
|
||||
@@ -41,19 +41,20 @@ Namespace API.Twitter
|
||||
Return My.Resources.SiteResources.TwitterPic_400
|
||||
End Get
|
||||
End Property
|
||||
#Region "Auth"
|
||||
<PropertyOption(AllowNull:=False, IsAuth:=True, ControlText:="Authorization",
|
||||
ControlToolTip:="Set authorization from [authorization] response header. This field must start from [Bearer] key word")>
|
||||
Private ReadOnly Property Auth As PropertyValue
|
||||
<PropertyOption(AllowNull:=False, IsAuth:=True, ControlText:="Token", ControlToolTip:="Set token from [x-csrf-token] response header")>
|
||||
Private ReadOnly Property Token As PropertyValue
|
||||
#End Region
|
||||
'TODELETE: twitter headers
|
||||
'#Region "Auth"
|
||||
' <PropertyOption(AllowNull:=False, IsAuth:=False, ControlText:="Authorization",
|
||||
' ControlToolTip:="Set authorization from [authorization] response header. This field must start from [Bearer] key word")>
|
||||
' Private ReadOnly Property Auth As PropertyValue
|
||||
' <PropertyOption(AllowNull:=False, IsAuth:=False, ControlText:="Token", ControlToolTip:="Set token from [x-csrf-token] response header")>
|
||||
' Private ReadOnly Property Token As PropertyValue
|
||||
'#End Region
|
||||
#Region "Other properties"
|
||||
<PropertyOption(IsAuth:=False, ControlText:=GifsDownload_Text), PXML>
|
||||
<PropertyOption(ControlText:=GifsDownload_Text), PXML>
|
||||
Friend ReadOnly Property GifsDownload As PropertyValue
|
||||
<PropertyOption(IsAuth:=False, ControlText:=GifsSpecialFolder_Text, ControlToolTip:=GifsSpecialFolder_ToolTip), PXML>
|
||||
<PropertyOption(ControlText:=GifsSpecialFolder_Text, ControlToolTip:=GifsSpecialFolder_ToolTip), PXML>
|
||||
Friend ReadOnly Property GifsSpecialFolder As PropertyValue
|
||||
<PropertyOption(IsAuth:=False, ControlText:=GifsPrefix_Text, ControlToolTip:=GifsPrefix_ToolTip), PXML>
|
||||
<PropertyOption(ControlText:=GifsPrefix_Text, ControlToolTip:=GifsPrefix_ToolTip), PXML>
|
||||
Friend ReadOnly Property GifsPrefix As PropertyValue
|
||||
<Provider(NameOf(GifsSpecialFolder), Interaction:=True), Provider(NameOf(GifsPrefix), Interaction:=True)>
|
||||
Private ReadOnly Property GifStringChecker As IFormatProvider
|
||||
@@ -75,68 +76,52 @@ Namespace API.Twitter
|
||||
Throw New NotImplementedException("[GetFormat] is not available in the context of [GifStringProvider]")
|
||||
End Function
|
||||
End Class
|
||||
<PropertyOption(IsAuth:=False, ControlText:=UseMD5Comparison_Text, ControlToolTip:=UseMD5Comparison_ToolTip), PXML>
|
||||
<PropertyOption(ControlText:=UseMD5Comparison_Text, ControlToolTip:=UseMD5Comparison_ToolTip), PXML>
|
||||
Friend ReadOnly Property UseMD5Comparison As PropertyValue
|
||||
<PXML, PropertyOption(ControlText:="Concurrent downloads", ControlToolTip:="The number of concurrent downloads.", LeftOffset:=120), TaskCounter>
|
||||
Friend ReadOnly Property ConcurrentDownloads As PropertyValue
|
||||
#End Region
|
||||
Friend Overrides ReadOnly Property Responser As Responser
|
||||
Private Sub ChangeResponserFields(ByVal PropName As String, ByVal Value As Object)
|
||||
If Not PropName.IsEmptyString Then
|
||||
Dim f$ = String.Empty
|
||||
Select Case PropName
|
||||
Case NameOf(Auth) : f = Header_Authorization
|
||||
Case NameOf(Token) : f = Header_Token
|
||||
End Select
|
||||
If Not f.IsEmptyString Then
|
||||
Responser.Headers.Remove(f)
|
||||
If Not CStr(Value).IsEmptyString Then Responser.Headers.Add(f, CStr(Value))
|
||||
Responser.SaveSettings()
|
||||
End If
|
||||
End If
|
||||
End Sub
|
||||
'TODELETE: twitter headers
|
||||
'Private Sub ChangeResponserFields(ByVal PropName As String, ByVal Value As Object)
|
||||
' If Not PropName.IsEmptyString Then
|
||||
' Dim f$ = String.Empty
|
||||
' Select Case PropName
|
||||
' Case NameOf(Auth) : f = Header_Authorization
|
||||
' Case NameOf(Token) : f = Header_Token
|
||||
' End Select
|
||||
' If Not f.IsEmptyString Then
|
||||
' Responser.Headers.Remove(f)
|
||||
' If Not CStr(Value).IsEmptyString Then Responser.Headers.Add(f, CStr(Value))
|
||||
' Responser.SaveSettings()
|
||||
' End If
|
||||
' End If
|
||||
'End Sub
|
||||
#End Region
|
||||
Friend Sub New()
|
||||
MyBase.New(TwitterSite)
|
||||
Responser = New Responser($"{SettingsFolderName}\Responser_{Site}.xml") With {.DeclaredError = EDP.ThrowException}
|
||||
MyBase.New(TwitterSite, "twitter.com")
|
||||
|
||||
Dim a$ = String.Empty
|
||||
Dim t$ = String.Empty
|
||||
'TODELETE: twitter headers
|
||||
'Dim a$ = String.Empty
|
||||
'Dim t$ = String.Empty
|
||||
|
||||
With Responser
|
||||
If .File.Exists Then
|
||||
.CookiesDomain = "twitter.com"
|
||||
.CookiesEncryptKey = SettingsCLS.CookieEncryptKey
|
||||
.LoadSettings()
|
||||
a = .Headers.Value(Header_Authorization)
|
||||
t = .Headers.Value(Header_Token)
|
||||
Else
|
||||
.ContentType = "application/json"
|
||||
.Accept = "*/*"
|
||||
.CookiesDomain = "twitter.com"
|
||||
.CookiesEncryptKey = SettingsCLS.CookieEncryptKey
|
||||
.Decoders.Add(SymbolsConverter.Converters.Unicode)
|
||||
.Headers.Add("sec-ch-ua", """Chromium"";v=""112"", ""Google Chrome"";v=""112"", ""Not:A-Brand"";v=""99""")
|
||||
.Headers.Add("sec-ch-ua-mobile", "?0")
|
||||
.Headers.Add("sec-fetch-dest", "empty")
|
||||
.Headers.Add("sec-fetch-mode", "cors")
|
||||
.Headers.Add("sec-fetch-site", "same-origin")
|
||||
.Headers.Add(Header_Token, String.Empty)
|
||||
.Headers.Add("x-twitter-active-user", "yes")
|
||||
.Headers.Add("x-twitter-auth-type", "OAuth2Session")
|
||||
.Headers.Add(Header_Authorization, String.Empty)
|
||||
.SaveSettings()
|
||||
End If
|
||||
'TODELETE: twitter headers
|
||||
'a = .Headers.Value(Header_Authorization)
|
||||
't = .Headers.Value(Header_Token)
|
||||
.Cookies.ChangedAllowInternalDrop = False
|
||||
.Cookies.Changed = False
|
||||
End With
|
||||
|
||||
Auth = New PropertyValue(a, GetType(String), Sub(v) ChangeResponserFields(NameOf(Auth), v))
|
||||
Token = New PropertyValue(t, GetType(String), Sub(v) ChangeResponserFields(NameOf(Token), v))
|
||||
'TODELETE: twitter headers
|
||||
'Auth = New PropertyValue(a, GetType(String), Sub(v) ChangeResponserFields(NameOf(Auth), v))
|
||||
'Token = New PropertyValue(t, GetType(String), Sub(v) ChangeResponserFields(NameOf(Token), v))
|
||||
|
||||
GifsDownload = New PropertyValue(True)
|
||||
GifsSpecialFolder = New PropertyValue(String.Empty, GetType(String))
|
||||
GifsPrefix = New PropertyValue("GIF_")
|
||||
GifStringChecker = New GifStringProvider
|
||||
UseMD5Comparison = New PropertyValue(False)
|
||||
ConcurrentDownloads = New PropertyValue(1)
|
||||
|
||||
UserRegex = RParams.DMS("[htps:/]{7,8}.*?twitter.com/([^/]+)", 1)
|
||||
UrlPatternUser = "https://twitter.com/{0}"
|
||||
@@ -151,18 +136,10 @@ Namespace API.Twitter
|
||||
Return $"https://twitter.com/{User.Name}/status/{Media.Post.ID}"
|
||||
End Function
|
||||
Friend Overrides Function BaseAuthExists() As Boolean
|
||||
Return Responser.CookiesExists And ACheck(Token.Value) And ACheck(Auth.Value)
|
||||
Return Responser.CookiesExists
|
||||
End Function
|
||||
Friend Overrides Function Available(ByVal What As ISiteSettings.Download, ByVal Silent As Boolean) As Boolean
|
||||
If MyBase.Available(What, Silent) Then
|
||||
If What = ISiteSettings.Download.SavedPosts Then
|
||||
Return Settings.GalleryDLFile.Exists
|
||||
Else
|
||||
Return True
|
||||
End If
|
||||
Else
|
||||
Return False
|
||||
End If
|
||||
Return Settings.GalleryDLFile.Exists And BaseAuthExists()
|
||||
End Function
|
||||
Friend Overrides Sub UserOptions(ByRef Options As Object, ByVal OpenForm As Boolean)
|
||||
If Options Is Nothing OrElse (Not TypeOf Options Is EditorExchangeOptions OrElse
|
||||
|
||||
@@ -6,19 +6,17 @@
|
||||
'
|
||||
' This program is distributed in the hope that it will be useful,
|
||||
' but WITHOUT ANY WARRANTY
|
||||
Imports System.Net
|
||||
Imports System.Threading
|
||||
Imports SCrawler.API.Base
|
||||
Imports SCrawler.API.YouTube.Objects
|
||||
Imports PersonalUtilities.Functions.XML
|
||||
Imports PersonalUtilities.Functions.RegularExpressions
|
||||
Imports PersonalUtilities.Tools.Web.Clients
|
||||
Imports PersonalUtilities.Tools
|
||||
Imports PersonalUtilities.Tools.Web.Documents.JSON
|
||||
Imports UStates = SCrawler.API.Base.UserMedia.States
|
||||
Imports UTypes = SCrawler.API.Base.UserMedia.Types
|
||||
Namespace API.Twitter
|
||||
Friend Class UserData : Inherits UserDataBase
|
||||
Protected SinglePostUrl As String = "https://api.twitter.com/1.1/statuses/show.json?id={0}&tweet_mode=extended"
|
||||
#Region "XML names"
|
||||
Private Const Name_GifsDownload As String = "GifsDownload"
|
||||
Private Const Name_GifsSpecialFolder As String = "GifsSpecialFolder"
|
||||
@@ -29,6 +27,19 @@ Namespace API.Twitter
|
||||
Friend Property GifsSpecialFolder As String = String.Empty
|
||||
Friend Property GifsPrefix As String = String.Empty
|
||||
Private ReadOnly _DataNames As List(Of String)
|
||||
Private ReadOnly Property MySettings As SiteSettings
|
||||
Get
|
||||
Return HOST.Source
|
||||
End Get
|
||||
End Property
|
||||
Private FileNameProvider As ANumbers = Nothing
|
||||
Private Sub ResetFileNameProvider(Optional ByVal GroupSize As Integer? = Nothing)
|
||||
FileNameProvider = New ANumbers With {.FormatOptions = ANumbers.Options.FormatNumberGroup + ANumbers.Options.Groups}
|
||||
FileNameProvider.GroupSize = If(GroupSize, 3)
|
||||
End Sub
|
||||
Private Function RenameGdlFile(ByVal Input As SFile, ByVal i As Integer) As SFile
|
||||
Return SFile.Rename(Input, $"{Input.PathWithSeparator}{i.NumToString(FileNameProvider)}.{Input.Extension}",, EDP.ThrowException)
|
||||
End Function
|
||||
#End Region
|
||||
#Region "Exchange options"
|
||||
Friend Overrides Function ExchangeOptionsGet() As Object
|
||||
@@ -79,134 +90,157 @@ Namespace API.Twitter
|
||||
DownloadData_SavedPosts(Token)
|
||||
Else
|
||||
If _ContentList.Count > 0 Then _DataNames.ListAddList(_ContentList.Select(Function(c) c.File.File), LAP.ClearBeforeAdd, LAP.NotContainsOnly)
|
||||
DownloadData(String.Empty, Token)
|
||||
DownloadData_Timeline(Token)
|
||||
End If
|
||||
End Sub
|
||||
Private Overloads Sub DownloadData(ByVal POST As String, ByVal Token As CancellationToken)
|
||||
Private Sub DownloadData_Timeline(ByVal Token As CancellationToken)
|
||||
Dim URL$ = String.Empty
|
||||
Dim tCache As CacheKeeper = Nothing
|
||||
Try
|
||||
Dim PostID$ = String.Empty
|
||||
Dim PostDate$
|
||||
Dim PostDate$, tmpUserId$
|
||||
Dim j As EContainer
|
||||
Dim nn As EContainer
|
||||
Dim NewPostDetected As Boolean = False
|
||||
Dim ExistsDetected As Boolean = False
|
||||
|
||||
Dim UID As Func(Of EContainer, String) = Function(e) e.XmlIfNothing.Item({"user", "id"}).XmlIfNothingValue
|
||||
|
||||
URL = $"https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name={Name}&count=200&exclude_replies=false&include_rts=1&tweet_mode=extended"
|
||||
If Not POST.IsEmptyString Then URL &= $"&max_id={POST}"
|
||||
|
||||
ThrowAny(Token)
|
||||
Dim r$ = Responser.GetResponse(URL)
|
||||
If Not r.IsEmptyString Then
|
||||
Using w As EContainer = JsonDocument.Parse(r)
|
||||
If w.ListExists Then
|
||||
|
||||
If POST.IsEmptyString And Not w.ItemF({0, "user"}) Is Nothing Then
|
||||
With w.ItemF({0, "user"})
|
||||
If .Value("screen_name").StringToLower = Name.ToLower Then
|
||||
UserSiteNameUpdate(.Value("name"))
|
||||
UserDescriptionUpdate(.Value("description"))
|
||||
Dim __getImage As Action(Of String) = Sub(ByVal img As String)
|
||||
If Not img.IsEmptyString Then
|
||||
Dim __imgFile As SFile = UrlFile(img, True)
|
||||
If Not __imgFile.Name.IsEmptyString Then
|
||||
If __imgFile.Extension.IsEmptyString Then __imgFile.Extension = "jpg"
|
||||
__imgFile.Path = MyFile.CutPath.Path
|
||||
If Not __imgFile.Exists Then GetWebFile(img, __imgFile, EDP.None)
|
||||
End If
|
||||
End If
|
||||
End Sub
|
||||
Dim icon$ = .Value("profile_image_url_https")
|
||||
If Not icon.IsEmptyString Then icon = icon.Replace("_normal", String.Empty)
|
||||
__getImage.Invoke(.Value("profile_banner_url"))
|
||||
__getImage.Invoke(icon)
|
||||
tCache = New CacheKeeper($"{DownloadContentDefault_GetRootDir()}\_tCache\")
|
||||
If tCache.RootDirectory.Exists(SFO.Path, False) Then tCache.RootDirectory.Delete(SFO.Path, SFODelete.DeletePermanently, EDP.ReturnValue)
|
||||
tCache.Validate()
|
||||
Dim f As SFile = GetTimelineFromGalleryDL(tCache.RootDirectory, Token)
|
||||
If Not f.IsEmptyString Then
|
||||
ThrowAny(Token)
|
||||
Dim timelineFiles As List(Of SFile) = SFile.GetFiles(f, "*.txt",, EDP.ReturnValue)
|
||||
If timelineFiles.ListExists Then
|
||||
Dim i%
|
||||
ResetFileNameProvider(Math.Max(timelineFiles.Count.ToString.Length, 2))
|
||||
'rename files
|
||||
For i = 0 To timelineFiles.Count - 1 : timelineFiles(i) = RenameGdlFile(timelineFiles(i), i) : Next
|
||||
'parse files
|
||||
For i = 0 To timelineFiles.Count - 1
|
||||
j = JsonDocument.Parse(timelineFiles(i).GetText)
|
||||
If Not j Is Nothing Then
|
||||
If i = 0 Then
|
||||
Dim resValue$ = j.Value({"data", "user", "result"}, "__typename").StringTrim.StringToLower
|
||||
If resValue.IsEmptyString Then
|
||||
UserExists = False
|
||||
j.Dispose()
|
||||
Exit Sub
|
||||
ElseIf resValue = "userunavailable" Then
|
||||
UserSuspended = True
|
||||
j.Dispose()
|
||||
Exit Sub
|
||||
Else
|
||||
With j({"data", "user", "result"})
|
||||
If .ListExists Then
|
||||
If ID.IsEmptyString Then
|
||||
ID = .Value("rest_id")
|
||||
If Not ID.IsEmptyString Then _ForceSaveUserInfo = True
|
||||
End If
|
||||
With .Item({"legacy"})
|
||||
If .ListExists Then
|
||||
If .Value("screen_name").StringToLower = Name.ToLower Then
|
||||
UserSiteNameUpdate(.Value("name"))
|
||||
UserDescriptionUpdate(.Value("description"))
|
||||
Dim __getImage As Action(Of String) = Sub(ByVal img As String)
|
||||
If Not img.IsEmptyString Then
|
||||
Dim __imgFile As SFile = UrlFile(img, True)
|
||||
If Not __imgFile.Name.IsEmptyString Then
|
||||
If __imgFile.Extension.IsEmptyString Then __imgFile.Extension = "jpg"
|
||||
__imgFile.Path = MyFile.CutPath.Path
|
||||
If Not __imgFile.Exists Then GetWebFile(img, __imgFile, EDP.None)
|
||||
If __imgFile.Exists Then IconBannerDownloaded = True
|
||||
End If
|
||||
End If
|
||||
End Sub
|
||||
Dim icon$ = .Value("profile_image_url_https")
|
||||
If Not icon.IsEmptyString Then icon = icon.Replace("_normal", String.Empty)
|
||||
If DownloadIconBanner Then
|
||||
__getImage.Invoke(.Value("profile_banner_url"))
|
||||
__getImage.Invoke(icon)
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
End With
|
||||
Else
|
||||
With j({"globalObjects", "tweets"})
|
||||
If .ListExists Then
|
||||
ProgressPre.ChangeMax(.Count)
|
||||
For Each nn In .Self
|
||||
ProgressPre.Perform()
|
||||
If nn.Count > 0 Then
|
||||
PostID = nn.Value("id")
|
||||
|
||||
'Date Pattern:
|
||||
'Sat Jan 01 01:10:15 +0000 2000
|
||||
If nn.Contains("created_at") Then PostDate = nn("created_at").Value Else PostDate = String.Empty
|
||||
Select Case CheckDatesLimit(PostDate, Declarations.DateProvider)
|
||||
Case DateResult.Skip : Continue For
|
||||
Case DateResult.Exit : Exit Sub
|
||||
End Select
|
||||
|
||||
If Not _TempPostsList.Contains(PostID) Then
|
||||
NewPostDetected = True
|
||||
_TempPostsList.Add(PostID)
|
||||
Else
|
||||
ExistsDetected = True
|
||||
Continue For
|
||||
End If
|
||||
|
||||
tmpUserId = nn.ItemF({"extended_entities", "media", 0, "source_user_id"}).
|
||||
XmlIfNothingValue.IfNullOrEmpty(nn.Value("user_id")).IfNullOrEmpty("/")
|
||||
|
||||
If Not ParseUserMediaOnly OrElse (Not ID.IsEmptyString AndAlso tmpUserId = ID) Then _
|
||||
ObtainMedia(nn, PostID, PostDate)
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
j.Dispose()
|
||||
End If
|
||||
|
||||
With If(IsSavedPosts, w({"globalObjects", "tweets"}).XmlIfNothing, w)
|
||||
ProgressPre.ChangeMax(.Count)
|
||||
For Each nn In .Self
|
||||
ProgressPre.Perform()
|
||||
ThrowAny(Token)
|
||||
If nn.Count > 0 Then
|
||||
PostID = nn.Value("id")
|
||||
If ID.IsEmptyString Then
|
||||
ID = UID(nn)
|
||||
If Not ID.IsEmptyString Then UpdateUserInformation()
|
||||
End If
|
||||
|
||||
'Date Pattern:
|
||||
'Sat Jan 01 01:10:15 +0000 2000
|
||||
If nn.Contains("created_at") Then PostDate = nn("created_at").Value Else PostDate = String.Empty
|
||||
Select Case CheckDatesLimit(PostDate, Declarations.DateProvider)
|
||||
Case DateResult.Skip : Continue For
|
||||
Case DateResult.Exit : Exit Sub
|
||||
End Select
|
||||
|
||||
If Not _TempPostsList.Contains(PostID) Then
|
||||
NewPostDetected = True
|
||||
_TempPostsList.Add(PostID)
|
||||
Else
|
||||
ExistsDetected = True
|
||||
Continue For
|
||||
End If
|
||||
|
||||
If Not ParseUserMediaOnly OrElse
|
||||
(Not nn.Contains("retweeted_status") OrElse (Not ID.IsEmptyString AndAlso UID(nn("retweeted_status")) = ID)) Then _
|
||||
ObtainMedia(nn, PostID, PostDate)
|
||||
End If
|
||||
Next
|
||||
End With
|
||||
End If
|
||||
End Using
|
||||
|
||||
If POST.IsEmptyString And ExistsDetected Then Exit Sub
|
||||
If Not PostID.IsEmptyString And NewPostDetected Then DownloadData(PostID, Token)
|
||||
Next
|
||||
End If
|
||||
End If
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, $"data downloading error [{URL}]")
|
||||
Finally
|
||||
If Not tCache Is Nothing Then tCache.Dispose()
|
||||
End Try
|
||||
End Sub
|
||||
Private Sub DownloadData_SavedPosts(ByVal Token As CancellationToken)
|
||||
Try
|
||||
Dim urls As List(Of String) = GetBookmarksUrlsFromGalleryDL()
|
||||
If urls.ListExists Then
|
||||
Dim postIds As New List(Of String)
|
||||
Dim r$
|
||||
Dim f As SFile = GetDataFromGalleryDL("https://twitter.com/i/bookmarks", Settings.Cache, True, Token)
|
||||
Dim files As List(Of SFile) = SFile.GetFiles(f, "*.txt")
|
||||
If files.ListExists Then
|
||||
ResetFileNameProvider(Math.Max(files.Count.ToString.Length, 3))
|
||||
Dim id$
|
||||
Dim j As EContainer, jj As EContainer
|
||||
Dim jErr As New ErrorsDescriber(EDP.ReturnValue)
|
||||
Dim rPattern As RParams = RParams.DM("(?<=tweet-)(\d+)\Z", 0, EDP.ReturnValue)
|
||||
ProgressPre.ChangeMax(urls.Count)
|
||||
For Each url$ In urls
|
||||
ProgressPre.Perform()
|
||||
r = Responser.GetResponse(url)
|
||||
If Not r.IsEmptyString Then
|
||||
j = JsonDocument.Parse(r, jErr)
|
||||
If Not j Is Nothing Then
|
||||
jj = j.ItemF({"data", "bookmark_timeline_v2", "timeline", "instructions", 0, "entries"})
|
||||
If If(jj?.Count, 0) > 0 Then postIds.ListAddList(jj.Select(Function(jj2) CStr(RegexReplace(jj2.Value("entryId"), rPattern))), LNC)
|
||||
j.Dispose()
|
||||
End If
|
||||
For i% = 0 To files.Count - 1
|
||||
f = RenameGdlFile(files(i), i)
|
||||
j = JsonDocument.Parse(f.GetText, jErr)
|
||||
If Not j Is Nothing Then
|
||||
With j.ItemF({"data", 0, "timeline", "instructions", 0, "entries"})
|
||||
If .ListExists Then
|
||||
ProgressPre.ChangeMax(.Count)
|
||||
For Each jj In .Self
|
||||
ProgressPre.Perform()
|
||||
With jj({"content", "itemContent", "tweet_results", "result", "legacy"})
|
||||
If .ListExists Then
|
||||
id = .Value("id_str")
|
||||
If _TempPostsList.Contains(id) Then j.Dispose() : Exit Sub Else ObtainMedia(.Self, id, .Value("created_at"))
|
||||
End If
|
||||
End With
|
||||
Next
|
||||
End If
|
||||
End With
|
||||
j.Dispose()
|
||||
End If
|
||||
Next
|
||||
If postIds.Count > 0 Then postIds.RemoveAll(Function(pid) pid.IsEmptyString OrElse (_TempPostsList.Contains(pid) Or _DataNames.Contains(pid)))
|
||||
If postIds.Count > 0 Then
|
||||
ProgressPre.ChangeMax(postIds.Count)
|
||||
For Each __id$ In postIds
|
||||
ProgressPre.Perform()
|
||||
_TempPostsList.Add(__id)
|
||||
r = Responser.GetResponse(String.Format(SinglePostUrl, __id),, EDP.ReturnValue)
|
||||
If Not r.IsEmptyString Then
|
||||
j = JsonDocument.Parse(r, jErr)
|
||||
If Not j Is Nothing Then
|
||||
If j.Count > 0 Then ObtainMedia(j, __id, j.Value("created_at"))
|
||||
j.Dispose()
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
End If
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, "data downloading error (Saved Posts)")
|
||||
@@ -215,21 +249,22 @@ Namespace API.Twitter
|
||||
#End Region
|
||||
#Region "Obtain media"
|
||||
Private Sub ObtainMedia(ByVal e As EContainer, ByVal PostID As String, ByVal PostDate As String, Optional ByVal State As UStates = UStates.Unknown)
|
||||
If Not CheckVideoNode(e, PostID, PostDate, State) Then
|
||||
Dim s As EContainer = e.ItemF({"extended_entities", "media"})
|
||||
If s Is Nothing OrElse s.Count = 0 Then s = e.ItemF({"retweeted_status", "extended_entities", "media"})
|
||||
If If(s?.Count, 0) > 0 Then
|
||||
For Each m In s
|
||||
If m.Contains("media_url") Then
|
||||
Dim dName$ = UrlFile(m("media_url").Value)
|
||||
Dim s As EContainer = e.ItemF({"extended_entities", "media"})
|
||||
If s Is Nothing OrElse s.Count = 0 Then s = e.ItemF({"retweeted_status", "extended_entities", "media"})
|
||||
If If(s?.Count, 0) > 0 Then
|
||||
Dim mUrl$
|
||||
For Each m As EContainer In s
|
||||
If Not CheckVideoNode(m, PostID, PostDate, State) Then
|
||||
mUrl = m.Value("media_url").IfNullOrEmpty(m.Value("media_url_https"))
|
||||
If Not mUrl.IsEmptyString Then
|
||||
Dim dName$ = UrlFile(mUrl)
|
||||
If Not dName.IsEmptyString AndAlso Not _DataNames.Contains(dName) Then
|
||||
_DataNames.Add(dName)
|
||||
_TempMediaList.ListAddValue(MediaFromData(m("media_url").Value,
|
||||
PostID, PostDate, GetPictureOption(m), State, UTypes.Picture), LNC)
|
||||
_TempMediaList.ListAddValue(MediaFromData(mUrl, PostID, PostDate, GetPictureOption(m), State, UTypes.Picture), LNC)
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
End Sub
|
||||
Private Function CheckVideoNode(ByVal w As EContainer, ByVal PostID As String, ByVal PostDate As String,
|
||||
@@ -260,34 +295,32 @@ Namespace API.Twitter
|
||||
Dim url$, ff$
|
||||
Dim f As SFile
|
||||
Dim m As UserMedia
|
||||
With w({"extended_entities", "media"})
|
||||
If .ListExists Then
|
||||
For Each n As EContainer In .Self
|
||||
If n.Value("type") = "animated_gif" Then
|
||||
With n({"video_info", "variants"})
|
||||
If .ListExists Then
|
||||
With .ItemF({gifUrl})
|
||||
If .ListExists Then
|
||||
url = .Value("url")
|
||||
ff = UrlFile(url)
|
||||
If Not ff.IsEmptyString Then
|
||||
If GifsDownload And Not _DataNames.Contains(ff) Then
|
||||
m = MediaFromData(url, PostID, PostDate,, State, UTypes.Video)
|
||||
f = m.File
|
||||
If Not f.IsEmptyString And Not GifsPrefix.IsEmptyString Then f.Name = $"{GifsPrefix}{f.Name}" : m.File = f
|
||||
If Not GifsSpecialFolder.IsEmptyString Then m.SpecialFolder = $"{GifsSpecialFolder}*"
|
||||
_TempMediaList.ListAddValue(m, LNC)
|
||||
End If
|
||||
Return True
|
||||
If w.ListExists Then
|
||||
For Each n As EContainer In w
|
||||
If n.Value("type") = "animated_gif" Then
|
||||
With n({"video_info", "variants"})
|
||||
If .ListExists Then
|
||||
With .ItemF({gifUrl})
|
||||
If .ListExists Then
|
||||
url = .Value("url")
|
||||
ff = UrlFile(url)
|
||||
If Not ff.IsEmptyString Then
|
||||
If GifsDownload And Not _DataNames.Contains(ff) Then
|
||||
m = MediaFromData(url, PostID, PostDate,, State, UTypes.Video)
|
||||
f = m.File
|
||||
If Not f.IsEmptyString And Not GifsPrefix.IsEmptyString Then f.Name = $"{GifsPrefix}{f.Name}" : m.File = f
|
||||
If Not GifsSpecialFolder.IsEmptyString Then m.SpecialFolder = $"{GifsSpecialFolder}*"
|
||||
_TempMediaList.ListAddValue(m, LNC)
|
||||
End If
|
||||
Return True
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
Return False
|
||||
Catch ex As Exception
|
||||
LogError(ex, "[API.Twitter.UserData.CheckForGif]")
|
||||
@@ -295,64 +328,153 @@ Namespace API.Twitter
|
||||
End Try
|
||||
End Function
|
||||
Private Function GetVideoNodeURL(ByVal w As EContainer) As String
|
||||
Dim v As EContainer = w.GetNode(VideoNode)
|
||||
If v.ListExists Then
|
||||
Dim l As New List(Of Sizes)
|
||||
Dim u$
|
||||
Dim nn As EContainer
|
||||
For Each n As EContainer In v
|
||||
If n.Count > 0 Then
|
||||
For Each nn In n
|
||||
If nn("content_type").XmlIfNothingValue("none").Contains("mp4") AndAlso nn.Contains("url") Then
|
||||
u = nn.Value("url")
|
||||
With w({"video_info", "variants"})
|
||||
If .ListExists Then
|
||||
Dim l As New List(Of Sizes)
|
||||
Dim u$
|
||||
For Each n As EContainer In .Self
|
||||
If n.Count > 0 Then
|
||||
If n("content_type").XmlIfNothingValue("none").Contains("mp4") AndAlso n.Contains("url") Then
|
||||
u = n.Value("url")
|
||||
l.Add(New Sizes(RegexReplace(u, VideoSizeRegEx), u))
|
||||
End If
|
||||
Next
|
||||
End If
|
||||
Next
|
||||
If l.Count > 0 Then l.RemoveAll(Function(s) s.HasError)
|
||||
If l.Count > 0 Then l.Sort() : Return l(0).Data
|
||||
End If
|
||||
End If
|
||||
Next
|
||||
If l.Count > 0 Then l.RemoveAll(Function(s) s.HasError)
|
||||
If l.Count > 0 Then l.Sort() : Return l(0).Data
|
||||
End If
|
||||
End With
|
||||
Return String.Empty
|
||||
End Function
|
||||
#End Region
|
||||
#Region "Gallery-DL Support"
|
||||
Private Function GetBookmarksUrlsFromGalleryDL() As List(Of String)
|
||||
Dim command$ = $"gallery-dl --verbose --simulate --cookies ""{DirectCast(HOST.Source, SiteSettings).CookiesNetscapeFile}"" https://twitter.com/i/bookmarks"
|
||||
Private Class TwitterGDL : Inherits GDL.GDLBatch
|
||||
Private Property Token As CancellationToken
|
||||
Friend Sub New(ByVal Dir As SFile, ByVal _Token As CancellationToken)
|
||||
MyBase.New
|
||||
Commands.Clear()
|
||||
ChangeDirectory(Dir)
|
||||
Token = _Token
|
||||
End Sub
|
||||
Protected Overrides Async Function Validate(ByVal Value As String) As Task
|
||||
If Not ProcessKilled AndAlso Await Task.Run(Function() Token.IsCancellationRequested OrElse IdExists(Value)) Then Kill()
|
||||
End Function
|
||||
Private Function IdExists(ByVal Value As String) As Boolean
|
||||
Try
|
||||
Value = Value.StringTrim
|
||||
If Not Value.IsEmptyString AndAlso (Value.StartsWith("*") Or Value.StartsWith(".\gallery-dl\")) Then
|
||||
Dim id$ = Value.Split("\").Last.Split(".").First.Split("_").First
|
||||
If Not id.IsEmptyString Then Return TempPostsList.Contains(id)
|
||||
End If
|
||||
Catch ex As Exception
|
||||
End Try
|
||||
Return False
|
||||
End Function
|
||||
End Class
|
||||
Private Function GetDataFromGalleryDL(ByVal URL As String, ByVal Cache As CacheKeeper, ByVal UseTempPostList As Boolean,
|
||||
Optional ByVal Token As CancellationToken = Nothing) As SFile
|
||||
Dim command$ = $"""{Settings.GalleryDLFile}"" --verbose --no-download --no-skip --cookies ""{MySettings.CookiesNetscapeFile}"" --write-pages "
|
||||
Try
|
||||
Using batch As New GDL.GDLBatch With {.TempPostsList = _TempPostsList} : Return GDL.GetUrlsFromGalleryDl(batch, command) : End Using
|
||||
Catch ex As Exception
|
||||
HasError = True
|
||||
LogError(ex, $"GetJson({command})")
|
||||
Dim dir As SFile = Cache.NewPath
|
||||
If dir.Exists(SFO.Path,, EDP.ThrowException) Then
|
||||
Using batch As New TwitterGDL(dir, Token)
|
||||
If UseTempPostList Then
|
||||
batch.TempPostsList = _TempPostsList
|
||||
command &= GdlGetIdFilterString()
|
||||
End If
|
||||
command &= URL
|
||||
'#If DEBUG Then
|
||||
' Debug.WriteLine(command)
|
||||
'#End If
|
||||
batch.Execute(command)
|
||||
End Using
|
||||
Return dir
|
||||
End If
|
||||
Return Nothing
|
||||
Catch ex As Exception
|
||||
Return ErrorsDescriber.Execute(EDP.SendToLog, ex, $"{ToStringForLog()}: GetDataFromGalleryDL({command})")
|
||||
End Try
|
||||
End Function
|
||||
Private Function GetTimelineFromGalleryDL(ByVal Cache As CacheKeeper, ByVal Token As CancellationToken) As SFile
|
||||
Dim command$ = String.Empty
|
||||
Try
|
||||
Dim conf As SFile = $"{Cache.NewPath.PathWithSeparator}TwitterGdlConfig.conf"
|
||||
Dim confText$ = "{""extractor"":{""cookies"": """ & MySettings.CookiesNetscapeFile.ToString.Replace("\", "/") &
|
||||
""",""cookies-update"": false,""twitter"":{""cards"": false,""conversations"": false,""pinned"": false,""quoted"": false,""replies"": true,""retweets"": true,""strategy"": null,""text-tweets"": false,""twitpic"": false,""unique"": true,""users"": ""timeline"",""videos"": true}}}"
|
||||
If conf.Exists(SFO.Path, True, EDP.ThrowException) Then TextSaver.SaveTextToFile(confText, conf)
|
||||
If Not conf.Exists Then Throw New IO.FileNotFoundException("Can't find Twitter GDL config file", conf)
|
||||
|
||||
command = $"""{Settings.GalleryDLFile}"" --verbose --no-download --no-skip --config ""{conf}"" --write-pages "
|
||||
command &= GdlGetIdFilterString()
|
||||
command &= $"https://twitter.com/search?q=from:{Name}+include:nativeretweets"
|
||||
Dim dir As SFile = Cache.NewPath
|
||||
dir.Exists(SFO.Path, True, EDP.ThrowException)
|
||||
'#If DEBUG Then
|
||||
' Debug.WriteLine(command)
|
||||
'#End If
|
||||
Using tgdl As New TwitterGDL(dir, Token) With {.TempPostsList = _TempPostsList} : tgdl.Execute(command) : End Using
|
||||
Return dir
|
||||
Catch ex As Exception
|
||||
Return ErrorsDescriber.Execute(EDP.SendToLog, ex, $"{ToStringForLog()}: GetTimelineFromGalleryDL({command})")
|
||||
End Try
|
||||
End Function
|
||||
Private Function GdlGetIdFilterString() As String
|
||||
Return If(_TempPostsList.Count > 0, $"--filter ""int(tweet_id) > {_TempPostsList.Last} or abort()"" ", String.Empty)
|
||||
End Function
|
||||
#End Region
|
||||
#Region "ReparseMissing"
|
||||
Protected Overrides Sub ReparseMissing(ByVal Token As CancellationToken)
|
||||
Const SinglePostPattern$ = "https://twitter.com/{0}/status/{1}"
|
||||
Dim rList As New List(Of Integer)
|
||||
Dim URL$ = String.Empty
|
||||
Dim cache As CacheKeeper = Nothing
|
||||
Try
|
||||
If ContentMissingExists Then
|
||||
Dim m As UserMedia
|
||||
Dim r$, PostDate$
|
||||
Dim PostDate$
|
||||
Dim j As EContainer
|
||||
Dim f As SFile
|
||||
Dim i%, ii%
|
||||
Dim files As List(Of SFile)
|
||||
ResetFileNameProvider()
|
||||
If IsSingleObjectDownload Then
|
||||
cache = Settings.Cache
|
||||
Else
|
||||
cache = New CacheKeeper(DownloadContentDefault_GetRootDir.CSFilePS)
|
||||
End If
|
||||
ProgressPre.ChangeMax(_ContentList.Count)
|
||||
For i% = 0 To _ContentList.Count - 1
|
||||
For i = 0 To _ContentList.Count - 1
|
||||
ProgressPre.Perform()
|
||||
If _ContentList(i).State = UStates.Missing Then
|
||||
m = _ContentList(i)
|
||||
If Not m.Post.ID.IsEmptyString Then
|
||||
If Not m.Post.ID.IsEmptyString Or (IsSingleObjectDownload And Not m.URL_BASE.IsEmptyString) Then
|
||||
ThrowAny(Token)
|
||||
URL = String.Format(SinglePostUrl, m.Post.ID)
|
||||
r = Responser.GetResponse(URL,, EDP.ReturnValue)
|
||||
If Not r.IsEmptyString Then
|
||||
j = JsonDocument.Parse(r)
|
||||
If Not j Is Nothing Then
|
||||
PostDate = String.Empty
|
||||
If j.Contains("created_at") Then PostDate = j("created_at").Value Else PostDate = String.Empty
|
||||
ObtainMedia(j, m.Post.ID, PostDate, UStates.Missing)
|
||||
rList.Add(i)
|
||||
If IsSingleObjectDownload Then
|
||||
URL = m.URL_BASE
|
||||
Else
|
||||
URL = String.Format(SinglePostPattern, Name, m.Post.ID)
|
||||
End If
|
||||
f = GetDataFromGalleryDL(URL, cache, Favorite, Token)
|
||||
If Not f.IsEmptyString Then
|
||||
files = SFile.GetFiles(f, "*.txt")
|
||||
If files.ListExists Then
|
||||
For ii = 0 To files.Count - 1
|
||||
f = RenameGdlFile(files(ii), ii)
|
||||
j = JsonDocument.Parse(f.GetText)
|
||||
If Not j Is Nothing Then
|
||||
With j.ItemF({"data", 0, "instructions", 0, "entries", 0,
|
||||
"content", "itemContent", "tweet_results", "result", "legacy"})
|
||||
If .ListExists Then
|
||||
PostDate = String.Empty
|
||||
If .Contains("created_at") Then PostDate = .Value("created_at") Else PostDate = String.Empty
|
||||
ObtainMedia(.Self, m.Post.ID, PostDate, UStates.Missing)
|
||||
rList.Add(i)
|
||||
End If
|
||||
End With
|
||||
j.Dispose()
|
||||
End If
|
||||
Next
|
||||
files.Clear()
|
||||
End If
|
||||
End If
|
||||
End If
|
||||
@@ -362,6 +484,7 @@ Namespace API.Twitter
|
||||
Catch ex As Exception
|
||||
ProcessException(ex, Token, $"ReparseMissing error [{URL}]")
|
||||
Finally
|
||||
If Not cache Is Nothing And Not IsSingleObjectDownload Then cache.Dispose()
|
||||
If rList.Count > 0 Then
|
||||
For i% = rList.Count - 1 To 0 Step -1 : _ContentList.RemoveAt(i) : Next
|
||||
rList.Clear()
|
||||
@@ -371,15 +494,8 @@ Namespace API.Twitter
|
||||
#End Region
|
||||
#Region "DownloadSingleObject"
|
||||
Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)
|
||||
Dim PostID$ = RegexReplace(Data.URL, RParams.DM("(?<=/)\d+", 0))
|
||||
If Not PostID.IsEmptyString Then
|
||||
Dim r$ = Responser.GetResponse(String.Format(SinglePostUrl, PostID),, EDP.ReturnValue)
|
||||
If Not r.IsEmptyString Then
|
||||
Using j As EContainer = JsonDocument.Parse(r)
|
||||
If j.ListExists Then ObtainMedia(j, j.Value("id"), j.Value("created_at"))
|
||||
End Using
|
||||
End If
|
||||
End If
|
||||
_ContentList.Add(New UserMedia(Data.URL) With {.State = UStates.Missing})
|
||||
ReparseMissing(Token)
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Picture options"
|
||||
@@ -459,26 +575,7 @@ Namespace API.Twitter
|
||||
#Region "Exception"
|
||||
Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False,
|
||||
Optional ByVal EObj As Object = Nothing) As Integer
|
||||
If AEquals(EObj, VALIDATE_MD5_ERROR) Then
|
||||
If Not FromPE Then LogError(ex, Message)
|
||||
Return 0
|
||||
Else
|
||||
With Responser
|
||||
If .StatusCode = HttpStatusCode.NotFound Then
|
||||
UserExists = False
|
||||
ElseIf .StatusCode = HttpStatusCode.Unauthorized Then
|
||||
UserSuspended = True
|
||||
ElseIf .StatusCode = HttpStatusCode.BadRequest Then
|
||||
MyMainLOG = "Twitter has invalid credentials"
|
||||
ElseIf .StatusCode = HttpStatusCode.ServiceUnavailable Or .StatusCode = HttpStatusCode.InternalServerError Then
|
||||
MyMainLOG = $"[{CInt(.StatusCode)}] Twitter is currently unavailable ({ToString()})"
|
||||
Else
|
||||
If Not FromPE Then LogError(ex, Message) : HasError = True
|
||||
Return 0
|
||||
End If
|
||||
End With
|
||||
End If
|
||||
Return 1
|
||||
Return 0
|
||||
End Function
|
||||
#End Region
|
||||
#Region "IDisposable support"
|
||||
|
||||
@@ -196,8 +196,10 @@ Namespace DownloadObjects
|
||||
End If
|
||||
End Sub
|
||||
Private Sub JobProgress_Progress0Changed(ByVal Sender As Object, ByVal e As ProgressEventArgs)
|
||||
MainProgress.Value0 = DirectCast(Sender, MyProgressExt).Value0
|
||||
MainProgress.Perform0(0)
|
||||
If Not Job.Type = Download.SavedPosts Then
|
||||
MainProgress.Value0 = DirectCast(Sender, MyProgressExt).Value0
|
||||
MainProgress.Perform0(0)
|
||||
End If
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "IDisposable Support"
|
||||
|
||||
@@ -43,7 +43,7 @@ Namespace DownloadObjects.STDownloader
|
||||
Return ErrorsDescriber.Execute(EDP.SendToLog, ex, "VideoListForm.LoadData_GetFiles", New List(Of IYouTubeMediaContainer))
|
||||
End Try
|
||||
End Function
|
||||
Protected Overrides Sub BTT_ADD_KeyClick(ByVal Sender As ToolStripMenuItemKeyClick, ByVal e As KeyClickEventArgs)
|
||||
Protected Overrides Sub BTT_ADD_KeyClick(ByVal Sender As ToolStripMenuItemKeyClick, ByVal e As KeyClickEventArgs) Handles BTT_ADD_URLS_ARR.KeyClick
|
||||
Dim __tag$ = UniversalFunctions.IfNullOrEmpty(Of Object)(Sender.Tag, String.Empty)
|
||||
If Not __tag = "a" And Not __tag = UrlsArrTag Then
|
||||
MyBase.BTT_ADD_KeyClick(Sender, e)
|
||||
@@ -51,6 +51,7 @@ Namespace DownloadObjects.STDownloader
|
||||
Dim url$ = String.Empty
|
||||
Try
|
||||
url = BufferText
|
||||
Dim disableDown As Boolean = e.Shift
|
||||
Dim output As SFile = Settings.LatestSavingPath
|
||||
Dim isArr As Boolean = __tag = UrlsArrTag
|
||||
Dim formOpened As Boolean = False
|
||||
@@ -152,7 +153,7 @@ Namespace DownloadObjects.STDownloader
|
||||
If media Is Nothing Then
|
||||
MsgBoxE({$"The URL you entered is not recognized by existing plugins.{vbCr}{url}", "Download video"}, vbCritical)
|
||||
Else
|
||||
ControlCreateAndAdd(media)
|
||||
ControlCreateAndAdd(media, disableDown)
|
||||
End If
|
||||
End If
|
||||
Catch ex As Exception
|
||||
|
||||
111
SCrawler/Editors/GlobalSettingsForm.Designer.vb
generated
111
SCrawler/Editors/GlobalSettingsForm.Designer.vb
generated
@@ -122,6 +122,8 @@ Namespace Editors
|
||||
Me.CH_NAME_SITE_FRIENDLY = New System.Windows.Forms.CheckBox()
|
||||
Me.CH_STD = New System.Windows.Forms.CheckBox()
|
||||
Me.CH_STD_EVERY = New System.Windows.Forms.CheckBox()
|
||||
Me.CH_STD_YT_LOAD = New System.Windows.Forms.CheckBox()
|
||||
Me.CH_STD_YT_REMOVE = New System.Windows.Forms.CheckBox()
|
||||
Me.TXT_CHANNELS_ROWS = New PersonalUtilities.Forms.Controls.TextBoxExtended()
|
||||
Me.TXT_CHANNELS_COLUMNS = New PersonalUtilities.Forms.Controls.TextBoxExtended()
|
||||
Me.CH_DOWN_IMAGES_NATIVE = New System.Windows.Forms.CheckBox()
|
||||
@@ -157,8 +159,7 @@ Namespace Editors
|
||||
Me.TAB_MAIN = New System.Windows.Forms.TabControl()
|
||||
Me.TAB_ENVIR = New System.Windows.Forms.TabPage()
|
||||
Me.CONTAINER_MAIN = New System.Windows.Forms.ToolStripContainer()
|
||||
Me.CH_STD_YT_LOAD = New System.Windows.Forms.CheckBox()
|
||||
Me.CH_STD_YT_REMOVE = New System.Windows.Forms.CheckBox()
|
||||
Me.CH_UICON_UP = New System.Windows.Forms.CheckBox()
|
||||
TP_BASIS = New System.Windows.Forms.TableLayoutPanel()
|
||||
TP_IMAGES = New System.Windows.Forms.TableLayoutPanel()
|
||||
TP_FILE_NAME = New System.Windows.Forms.TableLayoutPanel()
|
||||
@@ -508,13 +509,13 @@ Namespace Editors
|
||||
TP_FILE_NAME.Controls.Add(Me.OPT_FILE_NAME_ADD_DATE, 2, 0)
|
||||
TP_FILE_NAME.Controls.Add(Me.CH_FILE_NAME_CHANGE, 0, 0)
|
||||
TP_FILE_NAME.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
TP_FILE_NAME.Location = New System.Drawing.Point(1, 53)
|
||||
TP_FILE_NAME.Location = New System.Drawing.Point(1, 79)
|
||||
TP_FILE_NAME.Margin = New System.Windows.Forms.Padding(0)
|
||||
TP_FILE_NAME.Name = "TP_FILE_NAME"
|
||||
TP_FILE_NAME.RowCount = 1
|
||||
TP_FILE_NAME.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
|
||||
TP_FILE_NAME.Size = New System.Drawing.Size(574, 30)
|
||||
TP_FILE_NAME.TabIndex = 2
|
||||
TP_FILE_NAME.TabIndex = 3
|
||||
'
|
||||
'OPT_FILE_NAME_REPLACE
|
||||
'
|
||||
@@ -566,14 +567,14 @@ Namespace Editors
|
||||
TP_FILE_PATTERNS.Controls.Add(Me.OPT_FILE_DATE_START, 3, 0)
|
||||
TP_FILE_PATTERNS.Controls.Add(Me.OPT_FILE_DATE_END, 4, 0)
|
||||
TP_FILE_PATTERNS.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
TP_FILE_PATTERNS.Location = New System.Drawing.Point(1, 84)
|
||||
TP_FILE_PATTERNS.Location = New System.Drawing.Point(1, 110)
|
||||
TP_FILE_PATTERNS.Margin = New System.Windows.Forms.Padding(0)
|
||||
TP_FILE_PATTERNS.Name = "TP_FILE_PATTERNS"
|
||||
TP_FILE_PATTERNS.RowCount = 1
|
||||
TP_FILE_PATTERNS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
|
||||
TP_FILE_PATTERNS.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 29.0!))
|
||||
TP_FILE_PATTERNS.Size = New System.Drawing.Size(574, 30)
|
||||
TP_FILE_PATTERNS.TabIndex = 3
|
||||
TP_FILE_PATTERNS.TabIndex = 4
|
||||
'
|
||||
'CH_FILE_DATE
|
||||
'
|
||||
@@ -880,10 +881,10 @@ Namespace Editors
|
||||
'
|
||||
Me.CH_DOWN_REPARSE_MISSING.AutoSize = True
|
||||
Me.CH_DOWN_REPARSE_MISSING.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
Me.CH_DOWN_REPARSE_MISSING.Location = New System.Drawing.Point(4, 202)
|
||||
Me.CH_DOWN_REPARSE_MISSING.Location = New System.Drawing.Point(4, 228)
|
||||
Me.CH_DOWN_REPARSE_MISSING.Name = "CH_DOWN_REPARSE_MISSING"
|
||||
Me.CH_DOWN_REPARSE_MISSING.Size = New System.Drawing.Size(568, 19)
|
||||
Me.CH_DOWN_REPARSE_MISSING.TabIndex = 7
|
||||
Me.CH_DOWN_REPARSE_MISSING.TabIndex = 8
|
||||
Me.CH_DOWN_REPARSE_MISSING.Text = "Trying to download missing posts using regular download"
|
||||
TT_MAIN.SetToolTip(Me.CH_DOWN_REPARSE_MISSING, "If missing posts exist, the missing posts will attempt to be downloaded via user " &
|
||||
"download")
|
||||
@@ -925,6 +926,34 @@ Namespace Editors
|
||||
TT_MAIN.SetToolTip(Me.CH_STD_EVERY, "Show notifications when download in standalone downloader is complete")
|
||||
Me.CH_STD_EVERY.UseVisualStyleBackColor = True
|
||||
'
|
||||
'CH_STD_YT_LOAD
|
||||
'
|
||||
Me.CH_STD_YT_LOAD.AutoSize = True
|
||||
Me.CH_STD_YT_LOAD.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
Me.CH_STD_YT_LOAD.Location = New System.Drawing.Point(4, 166)
|
||||
Me.CH_STD_YT_LOAD.Name = "CH_STD_YT_LOAD"
|
||||
Me.CH_STD_YT_LOAD.Padding = New System.Windows.Forms.Padding(100, 0, 0, 0)
|
||||
Me.CH_STD_YT_LOAD.Size = New System.Drawing.Size(568, 19)
|
||||
Me.CH_STD_YT_LOAD.TabIndex = 6
|
||||
Me.CH_STD_YT_LOAD.Text = "Load downloaded YouTube videos to the form"
|
||||
TT_MAIN.SetToolTip(Me.CH_STD_YT_LOAD, "If checked, downloaded YouTube videos will be loaded to the form. Otherwise, all " &
|
||||
"downloaded data will be loaded to the form except YouTube data.")
|
||||
Me.CH_STD_YT_LOAD.UseVisualStyleBackColor = True
|
||||
'
|
||||
'CH_STD_YT_REMOVE
|
||||
'
|
||||
Me.CH_STD_YT_REMOVE.AutoSize = True
|
||||
Me.CH_STD_YT_REMOVE.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
Me.CH_STD_YT_REMOVE.Location = New System.Drawing.Point(4, 192)
|
||||
Me.CH_STD_YT_REMOVE.Name = "CH_STD_YT_REMOVE"
|
||||
Me.CH_STD_YT_REMOVE.Padding = New System.Windows.Forms.Padding(100, 0, 0, 0)
|
||||
Me.CH_STD_YT_REMOVE.Size = New System.Drawing.Size(568, 19)
|
||||
Me.CH_STD_YT_REMOVE.TabIndex = 7
|
||||
Me.CH_STD_YT_REMOVE.Text = "Clear YouTube videos when clearing the list"
|
||||
TT_MAIN.SetToolTip(Me.CH_STD_YT_REMOVE, "If checked, YouTube videos will also be removed from the list. This action will a" &
|
||||
"lso affect the standalone 'YouTubeDownloader' app.")
|
||||
Me.CH_STD_YT_REMOVE.UseVisualStyleBackColor = True
|
||||
'
|
||||
'TP_CHANNELS_IMGS
|
||||
'
|
||||
TP_CHANNELS_IMGS.ColumnCount = 2
|
||||
@@ -1265,18 +1294,20 @@ Namespace Editors
|
||||
TP_DOWNLOADING.CellBorderStyle = System.Windows.Forms.TableLayoutPanelCellBorderStyle.[Single]
|
||||
TP_DOWNLOADING.ColumnCount = 1
|
||||
TP_DOWNLOADING.ColumnStyles.Add(New System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
|
||||
TP_DOWNLOADING.Controls.Add(TP_FILE_NAME, 0, 2)
|
||||
TP_DOWNLOADING.Controls.Add(TP_FILE_PATTERNS, 0, 3)
|
||||
TP_DOWNLOADING.Controls.Add(Me.TXT_SCRIPT, 0, 4)
|
||||
TP_DOWNLOADING.Controls.Add(TP_FILE_NAME, 0, 3)
|
||||
TP_DOWNLOADING.Controls.Add(TP_FILE_PATTERNS, 0, 4)
|
||||
TP_DOWNLOADING.Controls.Add(Me.TXT_SCRIPT, 0, 5)
|
||||
TP_DOWNLOADING.Controls.Add(Me.CH_UDESCR_UP, 0, 0)
|
||||
TP_DOWNLOADING.Controls.Add(Me.TXT_DOWN_COMPLETE_SCRIPT, 0, 5)
|
||||
TP_DOWNLOADING.Controls.Add(TP_MISSING_DATA, 0, 6)
|
||||
TP_DOWNLOADING.Controls.Add(Me.CH_DOWN_REPARSE_MISSING, 0, 7)
|
||||
TP_DOWNLOADING.Controls.Add(Me.TXT_DOWN_COMPLETE_SCRIPT, 0, 6)
|
||||
TP_DOWNLOADING.Controls.Add(TP_MISSING_DATA, 0, 7)
|
||||
TP_DOWNLOADING.Controls.Add(Me.CH_DOWN_REPARSE_MISSING, 0, 8)
|
||||
TP_DOWNLOADING.Controls.Add(Me.CH_UNAME_UP, 0, 1)
|
||||
TP_DOWNLOADING.Controls.Add(Me.CH_UICON_UP, 0, 2)
|
||||
TP_DOWNLOADING.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
TP_DOWNLOADING.Location = New System.Drawing.Point(0, 0)
|
||||
TP_DOWNLOADING.Name = "TP_DOWNLOADING"
|
||||
TP_DOWNLOADING.RowCount = 9
|
||||
TP_DOWNLOADING.RowCount = 10
|
||||
TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
||||
TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
||||
TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
|
||||
TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 30.0!))
|
||||
@@ -1305,12 +1336,12 @@ Namespace Editors
|
||||
Me.TXT_SCRIPT.CaptionWidth = 120.0R
|
||||
Me.TXT_SCRIPT.ChangeControlsEnableOnCheckedChange = False
|
||||
Me.TXT_SCRIPT.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
Me.TXT_SCRIPT.Location = New System.Drawing.Point(4, 118)
|
||||
Me.TXT_SCRIPT.Location = New System.Drawing.Point(4, 144)
|
||||
Me.TXT_SCRIPT.Name = "TXT_SCRIPT"
|
||||
Me.TXT_SCRIPT.PlaceholderEnabled = True
|
||||
Me.TXT_SCRIPT.PlaceholderText = "Enter script path here..."
|
||||
Me.TXT_SCRIPT.Size = New System.Drawing.Size(568, 22)
|
||||
Me.TXT_SCRIPT.TabIndex = 4
|
||||
Me.TXT_SCRIPT.TabIndex = 5
|
||||
'
|
||||
'TXT_DOWN_COMPLETE_SCRIPT
|
||||
'
|
||||
@@ -1320,12 +1351,12 @@ Namespace Editors
|
||||
Me.TXT_DOWN_COMPLETE_SCRIPT.CaptionToolTipText = "This command will be executed after all downloads are completed"
|
||||
Me.TXT_DOWN_COMPLETE_SCRIPT.CaptionWidth = 120.0R
|
||||
Me.TXT_DOWN_COMPLETE_SCRIPT.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
Me.TXT_DOWN_COMPLETE_SCRIPT.Location = New System.Drawing.Point(4, 147)
|
||||
Me.TXT_DOWN_COMPLETE_SCRIPT.Location = New System.Drawing.Point(4, 173)
|
||||
Me.TXT_DOWN_COMPLETE_SCRIPT.Name = "TXT_DOWN_COMPLETE_SCRIPT"
|
||||
Me.TXT_DOWN_COMPLETE_SCRIPT.PlaceholderEnabled = True
|
||||
Me.TXT_DOWN_COMPLETE_SCRIPT.PlaceholderText = "Enter command here..."
|
||||
Me.TXT_DOWN_COMPLETE_SCRIPT.Size = New System.Drawing.Size(568, 22)
|
||||
Me.TXT_DOWN_COMPLETE_SCRIPT.TabIndex = 5
|
||||
Me.TXT_DOWN_COMPLETE_SCRIPT.TabIndex = 6
|
||||
'
|
||||
'TP_MISSING_DATA
|
||||
'
|
||||
@@ -1336,14 +1367,14 @@ Namespace Editors
|
||||
TP_MISSING_DATA.Controls.Add(Me.CH_ADD_MISSING_TO_LOG, 0, 0)
|
||||
TP_MISSING_DATA.Controls.Add(Me.CH_ADD_MISSING_ERROS_TO_LOG, 1, 0)
|
||||
TP_MISSING_DATA.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
TP_MISSING_DATA.Location = New System.Drawing.Point(1, 173)
|
||||
TP_MISSING_DATA.Location = New System.Drawing.Point(1, 199)
|
||||
TP_MISSING_DATA.Margin = New System.Windows.Forms.Padding(0)
|
||||
TP_MISSING_DATA.Name = "TP_MISSING_DATA"
|
||||
TP_MISSING_DATA.RowCount = 1
|
||||
TP_MISSING_DATA.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
|
||||
TP_MISSING_DATA.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 24.0!))
|
||||
TP_MISSING_DATA.Size = New System.Drawing.Size(574, 25)
|
||||
TP_MISSING_DATA.TabIndex = 6
|
||||
TP_MISSING_DATA.TabIndex = 7
|
||||
'
|
||||
'CH_UNAME_UP
|
||||
'
|
||||
@@ -1352,7 +1383,7 @@ Namespace Editors
|
||||
Me.CH_UNAME_UP.Location = New System.Drawing.Point(4, 30)
|
||||
Me.CH_UNAME_UP.Name = "CH_UNAME_UP"
|
||||
Me.CH_UNAME_UP.Size = New System.Drawing.Size(568, 19)
|
||||
Me.CH_UNAME_UP.TabIndex = 7
|
||||
Me.CH_UNAME_UP.TabIndex = 1
|
||||
Me.CH_UNAME_UP.Text = "Update user site name every time"
|
||||
Me.CH_UNAME_UP.UseVisualStyleBackColor = True
|
||||
'
|
||||
@@ -1843,33 +1874,16 @@ Namespace Editors
|
||||
Me.CONTAINER_MAIN.TabIndex = 0
|
||||
Me.CONTAINER_MAIN.TopToolStripPanelVisible = False
|
||||
'
|
||||
'CH_STD_YT_LOAD
|
||||
'CH_UICON_UP
|
||||
'
|
||||
Me.CH_STD_YT_LOAD.AutoSize = True
|
||||
Me.CH_STD_YT_LOAD.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
Me.CH_STD_YT_LOAD.Location = New System.Drawing.Point(4, 166)
|
||||
Me.CH_STD_YT_LOAD.Name = "CH_STD_YT_LOAD"
|
||||
Me.CH_STD_YT_LOAD.Padding = New System.Windows.Forms.Padding(100, 0, 0, 0)
|
||||
Me.CH_STD_YT_LOAD.Size = New System.Drawing.Size(568, 19)
|
||||
Me.CH_STD_YT_LOAD.TabIndex = 6
|
||||
Me.CH_STD_YT_LOAD.Text = "Load downloaded YouTube videos to the form"
|
||||
TT_MAIN.SetToolTip(Me.CH_STD_YT_LOAD, "If checked, downloaded YouTube videos will be loaded to the form. Otherwise, all " &
|
||||
"downloaded data will be loaded to the form except YouTube data.")
|
||||
Me.CH_STD_YT_LOAD.UseVisualStyleBackColor = True
|
||||
'
|
||||
'CH_STD_YT_REMOVE
|
||||
'
|
||||
Me.CH_STD_YT_REMOVE.AutoSize = True
|
||||
Me.CH_STD_YT_REMOVE.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
Me.CH_STD_YT_REMOVE.Location = New System.Drawing.Point(4, 192)
|
||||
Me.CH_STD_YT_REMOVE.Name = "CH_STD_YT_REMOVE"
|
||||
Me.CH_STD_YT_REMOVE.Padding = New System.Windows.Forms.Padding(100, 0, 0, 0)
|
||||
Me.CH_STD_YT_REMOVE.Size = New System.Drawing.Size(568, 19)
|
||||
Me.CH_STD_YT_REMOVE.TabIndex = 7
|
||||
Me.CH_STD_YT_REMOVE.Text = "Clear YouTube videos when clearing the list"
|
||||
TT_MAIN.SetToolTip(Me.CH_STD_YT_REMOVE, "If checked, YouTube videos will also be removed from the list. This action will a" &
|
||||
"lso affect the standalone 'YouTubeDownloader' app.")
|
||||
Me.CH_STD_YT_REMOVE.UseVisualStyleBackColor = True
|
||||
Me.CH_UICON_UP.AutoSize = True
|
||||
Me.CH_UICON_UP.Dock = System.Windows.Forms.DockStyle.Fill
|
||||
Me.CH_UICON_UP.Location = New System.Drawing.Point(4, 56)
|
||||
Me.CH_UICON_UP.Name = "CH_UICON_UP"
|
||||
Me.CH_UICON_UP.Size = New System.Drawing.Size(568, 19)
|
||||
Me.CH_UICON_UP.TabIndex = 2
|
||||
Me.CH_UICON_UP.Text = "Update user icon and banner every time (where supported)"
|
||||
Me.CH_UICON_UP.UseVisualStyleBackColor = True
|
||||
'
|
||||
'GlobalSettingsForm
|
||||
'
|
||||
@@ -2041,5 +2055,6 @@ Namespace Editors
|
||||
Private WithEvents CH_STD_UPDATE_YT_PATH As CheckBox
|
||||
Private WithEvents CH_STD_YT_LOAD As CheckBox
|
||||
Private WithEvents CH_STD_YT_REMOVE As CheckBox
|
||||
Private WithEvents CH_UICON_UP As CheckBox
|
||||
End Class
|
||||
End Namespace
|
||||
@@ -89,6 +89,7 @@ Namespace Editors
|
||||
'Downloading
|
||||
CH_UDESCR_UP.Checked = .UpdateUserDescriptionEveryTime
|
||||
CH_UNAME_UP.Checked = .UserSiteNameUpdateEveryTime
|
||||
CH_UICON_UP.Checked = .UpdateUserIconBannerEveryTime
|
||||
TXT_SCRIPT.Checked = .ScriptData.Attribute
|
||||
TXT_SCRIPT.Text = .ScriptData.Value
|
||||
TXT_DOWN_COMPLETE_SCRIPT.Text = .DownloadsCompleteCommand
|
||||
@@ -244,6 +245,7 @@ Namespace Editors
|
||||
'Downloading
|
||||
.UpdateUserDescriptionEveryTime.Value = CH_UDESCR_UP.Checked
|
||||
.UserSiteNameUpdateEveryTime.Value = CH_UNAME_UP.Checked
|
||||
.UpdateUserIconBannerEveryTime.Value = CH_UICON_UP.Checked
|
||||
.ScriptData.Value = TXT_SCRIPT.Text
|
||||
.ScriptData.Attribute.Value = TXT_SCRIPT.Checked
|
||||
.DownloadsCompleteCommand.Value = TXT_DOWN_COMPLETE_SCRIPT.Text
|
||||
|
||||
@@ -492,13 +492,23 @@ Namespace Editors
|
||||
If Not info.IsEmptyString Then MsgBoxE({info, "User information"})
|
||||
End Sub
|
||||
Private Sub CONTEXT_BTT_OPEN_FOLDER_Click(sender As Object, e As EventArgs) Handles CONTEXT_BTT_OPEN_FOLDER.Click
|
||||
Dim u As UserOpt = GetUserFromList()
|
||||
If Not u Is Nothing Then u.User.OpenFolder()
|
||||
OpenUserFolder()
|
||||
End Sub
|
||||
Private Sub CONTEXT_BTT_OPEN_SITE_Click(sender As Object, e As EventArgs) Handles CONTEXT_BTT_OPEN_SITE.Click
|
||||
Dim u As UserOpt = GetUserFromList()
|
||||
If Not u Is Nothing Then u.User.OpenSite()
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "List handlers"
|
||||
Private Sub LIST_DATA_MouseDoubleClick(sender As Object, e As MouseEventArgs) Handles LIST_DATA.MouseDoubleClick
|
||||
OpenUserFolder()
|
||||
End Sub
|
||||
#End Region
|
||||
#Region "Functions"
|
||||
Private Sub OpenUserFolder()
|
||||
Dim u As UserOpt = GetUserFromList()
|
||||
If Not u Is Nothing Then u.User.OpenFolder()
|
||||
End Sub
|
||||
#End Region
|
||||
End Class
|
||||
End Namespace
|
||||
@@ -122,6 +122,8 @@ Friend Class ListImagesLoader
|
||||
UserDataList.Clear()
|
||||
UpdateInProgress = False
|
||||
End If
|
||||
Else
|
||||
UpdateInProgress = False
|
||||
End If
|
||||
Else
|
||||
Dim t As New List(Of Task)
|
||||
|
||||
13
SCrawler/MainFrame.Designer.vb
generated
13
SCrawler/MainFrame.Designer.vb
generated
@@ -125,6 +125,7 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
|
||||
Me.BTT_TRAY_SHOW_HIDE = New System.Windows.Forms.ToolStripMenuItem()
|
||||
Me.BTT_TRAY_CLOSE = New System.Windows.Forms.ToolStripMenuItem()
|
||||
Me.BTT_TRAY_CLOSE_NO_SCRIPT = New System.Windows.Forms.ToolStripMenuItem()
|
||||
Me.BTT_TRAY_DOWNLOADER = New System.Windows.Forms.ToolStripMenuItem()
|
||||
SEP_1 = New System.Windows.Forms.ToolStripSeparator()
|
||||
SEP_2 = New System.Windows.Forms.ToolStripSeparator()
|
||||
CONTEXT_SEP_1 = New System.Windows.Forms.ToolStripSeparator()
|
||||
@@ -826,9 +827,9 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
|
||||
'
|
||||
'TRAY_CONTEXT
|
||||
'
|
||||
Me.TRAY_CONTEXT.Items.AddRange(New System.Windows.Forms.ToolStripItem() {Me.BTT_TRAY_PAUSE_AUTOMATION, Me.BTT_TRAY_SILENT_MODE, Me.BTT_TRAY_FEED_SHOW, Me.BTT_TRAY_CHANNELS, TRAY_SEP_1, Me.BTT_TRAY_SHOW_HIDE, TRAY_SEP_2, Me.BTT_TRAY_CLOSE, Me.BTT_TRAY_CLOSE_NO_SCRIPT})
|
||||
Me.TRAY_CONTEXT.Items.AddRange(New System.Windows.Forms.ToolStripItem() {Me.BTT_TRAY_PAUSE_AUTOMATION, Me.BTT_TRAY_SILENT_MODE, Me.BTT_TRAY_FEED_SHOW, Me.BTT_TRAY_CHANNELS, Me.BTT_TRAY_DOWNLOADER, TRAY_SEP_1, Me.BTT_TRAY_SHOW_HIDE, TRAY_SEP_2, Me.BTT_TRAY_CLOSE, Me.BTT_TRAY_CLOSE_NO_SCRIPT})
|
||||
Me.TRAY_CONTEXT.Name = "TRAY_CONTEXT"
|
||||
Me.TRAY_CONTEXT.Size = New System.Drawing.Size(171, 170)
|
||||
Me.TRAY_CONTEXT.Size = New System.Drawing.Size(171, 192)
|
||||
'
|
||||
'BTT_TRAY_PAUSE_AUTOMATION
|
||||
'
|
||||
@@ -892,6 +893,13 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
|
||||
Me.BTT_TRAY_CLOSE_NO_SCRIPT.ToolTipText = "Close the program without executing the script"
|
||||
Me.BTT_TRAY_CLOSE_NO_SCRIPT.Visible = False
|
||||
'
|
||||
'BTT_TRAY_DOWNLOADER
|
||||
'
|
||||
Me.BTT_TRAY_DOWNLOADER.Image = Global.SCrawler.My.Resources.Resources.ArrowDownPic_Blue_24
|
||||
Me.BTT_TRAY_DOWNLOADER.Name = "BTT_TRAY_DOWNLOADER"
|
||||
Me.BTT_TRAY_DOWNLOADER.Size = New System.Drawing.Size(170, 22)
|
||||
Me.BTT_TRAY_DOWNLOADER.Text = "Downloader"
|
||||
'
|
||||
'MainFrame
|
||||
'
|
||||
Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)
|
||||
@@ -996,4 +1004,5 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form
|
||||
Private WithEvents BTT_TRAY_FEED_SHOW As ToolStripMenuItem
|
||||
Friend WithEvents MENU_DOWN_ALL As ToolStripDropDownButton
|
||||
Private WithEvents BTT_TRAY_CHANNELS As ToolStripMenuItem
|
||||
Private WithEvents BTT_TRAY_DOWNLOADER As ToolStripMenuItem
|
||||
End Class
|
||||
@@ -452,9 +452,9 @@ CloseResume:
|
||||
Downloader.AddRange(Settings.GetUsers(UserExistsPredicate), e.IncludeInTheFeed)
|
||||
End Sub
|
||||
Private Sub BTT_DOWN_SITE_FULL_KeyClick(sender As Object, e As MyKeyEventArgs) Handles BTT_DOWN_SITE_FULL.KeyClick
|
||||
DownloadSiteFull(False, e.IncludeInTheFeed)
|
||||
DownloadSiteFull(False, e.IncludeInTheFeed, e.Shift)
|
||||
End Sub
|
||||
Private Sub DownloadSiteFull(ByVal ReadyForDownloadOnly As Boolean, ByVal IncludeInTheFeed As Boolean)
|
||||
Private Sub DownloadSiteFull(ByVal ReadyForDownloadOnly As Boolean, ByVal IncludeInTheFeed As Boolean, Optional ByVal IgnoreExists As Boolean = False)
|
||||
Using f As New SiteSelectionForm(Settings.LatestDownloadedSites.ValuesList)
|
||||
f.ShowDialog()
|
||||
If f.DialogResult = DialogResult.OK Then
|
||||
@@ -462,7 +462,7 @@ CloseResume:
|
||||
Settings.LatestDownloadedSites.AddRange(f.SelectedSites)
|
||||
Settings.LatestDownloadedSites.Update()
|
||||
If f.SelectedSites.Count > 0 Then
|
||||
Downloader.AddRange(Settings.GetUsers(Function(u) f.SelectedSites.Contains(u.Site) And u.Exists And
|
||||
Downloader.AddRange(Settings.GetUsers(Function(u) f.SelectedSites.Contains(u.Site) And (u.Exists Or IgnoreExists) And
|
||||
(Not ReadyForDownloadOnly Or u.ReadyForDownload)), IncludeInTheFeed)
|
||||
End If
|
||||
End If
|
||||
@@ -513,7 +513,7 @@ CloseResume:
|
||||
TrayIcon.ContextMenuStrip.Hide()
|
||||
MainFrameObj.PauseButtons.UpdatePauseButtons()
|
||||
End Sub
|
||||
Private Sub BTT_DOWN_VIDEO_Click(sender As Object, e As EventArgs) Handles BTT_DOWN_VIDEO.Click
|
||||
Private Sub BTT_DOWN_VIDEO_Click(sender As Object, e As EventArgs) Handles BTT_DOWN_VIDEO.Click, BTT_TRAY_DOWNLOADER.Click
|
||||
VideoDownloader.FormShow()
|
||||
End Sub
|
||||
Private Sub BTT_DOWN_STOP_Click(sender As Object, e As EventArgs) Handles BTT_DOWN_STOP.Click
|
||||
|
||||
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
|
||||
' by using the '*' as shown below:
|
||||
' <Assembly: AssemblyVersion("1.0.*")>
|
||||
|
||||
<Assembly: AssemblyVersion("2023.5.12.0")>
|
||||
<Assembly: AssemblyFileVersion("2023.5.12.0")>
|
||||
<Assembly: AssemblyVersion("2023.6.5.0")>
|
||||
<Assembly: AssemblyFileVersion("2023.6.5.0")>
|
||||
<Assembly: NeutralResourcesLanguage("en")>
|
||||
|
||||
@@ -23,6 +23,15 @@ Namespace Plugin.Hosts
|
||||
End Property
|
||||
Friend Property Instance As UserDataBase
|
||||
Friend ReadOnly Property ExternalSource As IDownloadableMedia = Nothing
|
||||
Public Overrides ReadOnly Property Exists As Boolean
|
||||
Get
|
||||
If SiteKey = API.YouTube.YouTubeSiteKey Then
|
||||
Return MyBase.Exists
|
||||
Else
|
||||
Return _Exists
|
||||
End If
|
||||
End Get
|
||||
End Property
|
||||
Public Overrides Property File As SFile
|
||||
Get
|
||||
Return _File
|
||||
@@ -128,7 +137,7 @@ Namespace Plugin.Hosts
|
||||
End Sub
|
||||
Public Overrides Sub Load(ByVal f As SFile)
|
||||
MyBase.Load(f)
|
||||
If _Exists Then _Exists = File.Exists
|
||||
If _Exists Then _Exists = Not MediaState = UserMediaStates.Downloaded OrElse File.Exists
|
||||
End Sub
|
||||
Public Overrides Sub Save()
|
||||
If FileSettings.IsEmptyString Then
|
||||
@@ -142,6 +151,9 @@ Namespace Plugin.Hosts
|
||||
x.Save(FileSettings)
|
||||
End Using
|
||||
End Sub
|
||||
Public Overrides Function GetHashCode() As Integer
|
||||
Return URL.GetHashCode
|
||||
End Function
|
||||
Protected Overrides Sub Dispose(ByVal disposing As Boolean)
|
||||
If Not disposedValue And disposing Then Instance.DisposeIfReady() : ExternalSource.DisposeIfReady(False)
|
||||
MyBase.Dispose(disposing)
|
||||
|
||||
@@ -238,6 +238,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
|
||||
FromChannelDownloadTopUse = New XMLValue(Of Boolean)("FromChannelDownloadTopUse", False, MyXML, n)
|
||||
FromChannelCopyImageToUser = New XMLValue(Of Boolean)("FromChannelCopyImageToUser", True, MyXML, n)
|
||||
UpdateUserDescriptionEveryTime = New XMLValue(Of Boolean)("UpdateUserDescriptionEveryTime", True, MyXML, n)
|
||||
UpdateUserIconBannerEveryTime = New XMLValue(Of Boolean)("UpdateUserIconBannerEveryTime", True, MyXML, n)
|
||||
ScriptData = New XMLValueAttribute(Of String, Boolean)("ScriptData", "Use",,, MyXML, n)
|
||||
|
||||
n = {"Users", "FileName"}
|
||||
@@ -720,6 +721,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
|
||||
Friend ReadOnly Property FromChannelDownloadTopUse As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property FromChannelCopyImageToUser As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property UpdateUserDescriptionEveryTime As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property UpdateUserIconBannerEveryTime As XMLValue(Of Boolean)
|
||||
#Region "File naming"
|
||||
Friend ReadOnly Property FileAddDateToFileName As XMLValue(Of Boolean)
|
||||
Friend ReadOnly Property FileAddTimeToFileName As XMLValue(Of Boolean)
|
||||
|
||||
Reference in New Issue
Block a user