Use split ranges to export all messages.

Telegram/SourceFiles/export/data/export_data_types.cpp

@@ -1089,6 +1089,8 @@ void FinalizeDialogsInfo(DialogsInfo &info, const Settings &settings) {
 			Unexpected("Type in ApiWrap::onlyMyMessages.");
 		}();
 		dialog.onlyMyMessages = ((settings.fullChats & setting) != setting);
+
+		ranges::reverse(dialog.splits);
 	}
 }
 

Telegram/SourceFiles/export/data/export_data_types.h

@@ -446,13 +446,15 @@ struct DialogInfo {
 	TimeId topMessageDate = 0;
 	PeerId peerId = 0;
 
+	// User messages splits which contained that dialog.
+	std::vector<int> splits;
+
 	// Filled after the whole dialogs list is accumulated.
 	bool onlyMyMessages = false;
 	QString relativePath;
 
 	// Filled when requesting dialog messages.
-	int messagesCount = 0;
-
+	std::vector<int> messagesCountPerSplit;
 };
 
 struct DialogsInfo {

Telegram/SourceFiles/export/export_api_wrap.cpp

@@ -15,6 +15,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 #include "base/value_ordering.h"
 #include "base/bytes.h"
 
+#include <set>
 #include <deque>
 
 namespace Export {
@@ -107,10 +108,12 @@ struct ApiWrap::StartProcess {
 
 	enum class Step {
 		UserpicsCount,
+		SplitRanges,
 		DialogsCount,
 		LeftChannelsCount,
 	};
 	std::deque<Step> steps;
+	int splitIndex = 0;
 	StartInfo info;
 };
 
@@ -160,23 +163,23 @@ struct ApiWrap::FileProgress {
 	int total = 0;
 };
 
-struct ApiWrap::LeftChannelsProcess {
+struct ApiWrap::ChatsProcess {
 	Fn<bool(int count)> progress;
 	FnMut<void(Data::DialogsInfo&&)> done;
 
 	Data::DialogsInfo info;
+	int processedCount = 0;
+	std::map<Data::PeerId, int> indexByPeer;
+};
 
+struct ApiWrap::LeftChannelsProcess : ChatsProcess {
 	int fullCount = 0;
 	int offset = 0;
 	bool finished = false;
 };
 
-struct ApiWrap::DialogsProcess {
-	Fn<bool(int count)> progress;
-	FnMut<void(Data::DialogsInfo&&)> done;
-
-	Data::DialogsInfo info;
-
+struct ApiWrap::DialogsProcess : ChatsProcess {
+	int splitIndexPlusOne = 0;
 	Data::TimeId offsetDate = 0;
 	int32 offsetId = 0;
 	MTPInputPeer offsetPeer = MTP_inputPeerEmpty();
@@ -190,7 +193,8 @@ struct ApiWrap::ChatProcess {
 	Fn<bool(Data::MessagesSlice&&)> handleSlice;
 	FnMut<void()> done;
 
-	int32 offsetId = 1;
+	int localSplitIndex = 0;
+	int32 largestIdPlusOne = 1;
 
 	Data::ParseMediaContext context;
 	base::optional<Data::MessagesSlice> slice;
@@ -234,12 +238,24 @@ auto ApiWrap::mainRequest(Request &&request) {
 
 	return std::move(_mtp.request(MTPInvokeWithTakeout<Request>(
 		MTP_long(*_takeoutId),
-		request
+		std::forward<Request>(request)
 	)).fail([=](RPCError &&result) {
 		error(std::move(result));
 	}).toDC(MTP::ShiftDcId(0, MTP::kExportDcShift)));
 }
 
+template <typename Request>
+auto ApiWrap::splitRequest(int index, Request &&request) {
+	Expects(index < _splits.size());
+
+	//if (index == _splits.size() - 1) {
+	//	return mainRequest(std::forward<Request>(request));
+	//}
+	return mainRequest(MTPInvokeWithMessagesRange<Request>(
+		_splits[index],
+		std::forward<Request>(request)));
+}
+
 auto ApiWrap::fileRequest(const Data::FileLocation &location, int offset) {
 	Expects(location.dcId != 0);
 	Expects(_takeoutId.has_value());
@@ -284,6 +300,9 @@ void ApiWrap::startExport(
 	if (_settings->types & Settings::Type::Userpics) {
 		_startProcess->steps.push_back(Step::UserpicsCount);
 	}
+	if (_settings->types & Settings::Type::NonChannelChatsMask) {
+		_startProcess->steps.push_back(Step::SplitRanges);
+	}
 	if (_settings->types & Settings::Type::AnyChatsMask) {
 		_startProcess->steps.push_back(Step::DialogsCount);
 	}
@@ -303,14 +322,17 @@ void ApiWrap::sendNextStartRequest() {
 		finishStartProcess();
 		return;
 	}
+	using Step = StartProcess::Step;
 	const auto step = steps.front();
 	steps.pop_front();
 	switch (step) {
-	case StartProcess::Step::UserpicsCount:
+	case Step::UserpicsCount:
 		return requestUserpicsCount();
-	case StartProcess::Step::DialogsCount:
+	case Step::SplitRanges:
+		return requestSplitRanges();
+	case Step::DialogsCount:
 		return requestDialogsCount();
-	case StartProcess::Step::LeftChannelsCount:
+	case Step::LeftChannelsCount:
 		return requestLeftChannelsCount();
 	}
 	Unexpected("Step in ApiWrap::sendNextStartRequest.");
@@ -339,10 +361,20 @@ void ApiWrap::requestUserpicsCount() {
 	}).send();
 }
 
+void ApiWrap::requestSplitRanges() {
+	Expects(_startProcess != nullptr);
+
+	mainRequest(MTPmessages_GetSplitRanges(
+	)).done([=](const MTPVector<MTPMessageRange> &result) {
+		_splits = result.v;
+		sendNextStartRequest();
+	}).send();
+}
+
 void ApiWrap::requestDialogsCount() {
 	Expects(_startProcess != nullptr);
 
-	mainRequest(MTPmessages_GetDialogs(
+	splitRequest(_startProcess->splitIndex, MTPmessages_GetDialogs(
 		MTP_flags(0),
 		MTP_int(0), // offset_date
 		MTP_int(0), // offset_id
@@ -352,14 +384,18 @@ void ApiWrap::requestDialogsCount() {
 		Expects(_settings != nullptr);
 		Expects(_startProcess != nullptr);
 
-		_startProcess->info.dialogsCount = result.match(
+		_startProcess->info.dialogsCount += result.match(
 		[](const MTPDmessages_dialogs &data) {
 			return int(data.vdialogs.v.size());
 		}, [](const MTPDmessages_dialogsSlice &data) {
 			return data.vcount.v;
 		});
 
+		if (++_startProcess->splitIndex >= _splits.size()) {
 			sendNextStartRequest();
+		} else {
+			requestDialogsCount();
+		}
 	}).send();
 }
 
@@ -390,6 +426,8 @@ void ApiWrap::requestLeftChannelsList(
 		FnMut<void(Data::DialogsInfo&&)> done) {
 	Expects(_leftChannelsProcess != nullptr);
 
+	validateSplits();
+
 	_leftChannelsProcess->progress = std::move(progress);
 	_leftChannelsProcess->done = std::move(done);
 	requestLeftChannelsSlice();
@@ -414,13 +452,24 @@ void ApiWrap::requestDialogsList(
 		FnMut<void(Data::DialogsInfo&&)> done) {
 	Expects(_dialogsProcess == nullptr);
 
+	validateSplits();
+
 	_dialogsProcess = std::make_unique<DialogsProcess>();
+	_dialogsProcess->splitIndexPlusOne = _splits.size();
 	_dialogsProcess->progress = std::move(progress);
 	_dialogsProcess->done = std::move(done);
 
 	requestDialogsSlice();
 }
 
+void ApiWrap::validateSplits() {
+	if (_splits.empty()) {
+		_splits.push_back(MTP_messageRange(
+			MTP_int(0),
+			MTP_int(std::numeric_limits<int>::max())));
+	}
+}
+
 void ApiWrap::startMainSession(FnMut<void()> done) {
 	const auto sizeLimit = _settings->media.sizeLimit;
 	const auto hasFiles = (_settings->media.types != 0) && (sizeLimit > 0);
@@ -699,11 +748,41 @@ void ApiWrap::requestMessages(
 	_chatProcess->handleSlice = std::move(slice);
 	_chatProcess->done = std::move(done);
 
-	requestMessagesSlice([=](int count) {
+	requestMessagesCount(0);
+}
+
+void ApiWrap::requestMessagesCount(int localSplitIndex) {
+	Expects(_chatProcess != nullptr);
+	Expects(localSplitIndex < _chatProcess->info.splits.size());
+
+	requestChatMessages(
+		_chatProcess->info.splits[localSplitIndex],
+		0, // offset_id
+		0, // add_offset
+		1, // limit
+		[=](const MTPmessages_Messages &result) {
 		Expects(_chatProcess != nullptr);
 
-		_chatProcess->info.messagesCount = count;
-		return _chatProcess->start(_chatProcess->info);
+		const auto count = result.match(
+		[](const MTPDmessages_messages &data) {
+			return data.vmessages.v.size();
+		}, [](const MTPDmessages_messagesSlice &data) {
+			return data.vcount.v;
+		}, [](const MTPDmessages_channelMessages &data) {
+			return data.vcount.v;
+		}, [](const MTPDmessages_messagesNotModified &data) {
+			return -1;
+		});
+		if (count < 0) {
+			error("Unexpected messagesNotModified received.");
+			return;
+		}
+		_chatProcess->info.messagesCountPerSplit[localSplitIndex] = count;
+		if (localSplitIndex + 1 < _chatProcess->info.splits.size()) {
+			requestMessagesCount(localSplitIndex + 1);
+		} else if (_chatProcess->start(_chatProcess->info)) {
+			requestMessagesSlice();
+		}
 	});
 }
 
@@ -727,14 +806,15 @@ void ApiWrap::cancelExportFast() {
 void ApiWrap::requestDialogsSlice() {
 	Expects(_dialogsProcess != nullptr);
 
-	mainRequest(MTPmessages_GetDialogs(
+	const auto splitIndex = _dialogsProcess->splitIndexPlusOne - 1;
+	splitRequest(splitIndex, MTPmessages_GetDialogs(
 		MTP_flags(0),
 		MTP_int(_dialogsProcess->offsetDate),
 		MTP_int(_dialogsProcess->offsetId),
 		_dialogsProcess->offsetPeer,
 		MTP_int(kChatsSliceLimit)
 	)).done([=](const MTPmessages_Dialogs &result) {
-		const auto finished = result.match(
+		auto finished = result.match(
 		[](const MTPDmessages_dialogs &data) {
 			return true;
 		}, [](const MTPDmessages_dialogsSlice &data) {
@@ -742,30 +822,40 @@ void ApiWrap::requestDialogsSlice() {
 		});
 
 		auto info = Data::ParseDialogsInfo(result);
-		if (finished || info.list.empty()) {
-			finishDialogsList();
-		} else {
-			const auto &last = info.list.back();
-			_dialogsProcess->offsetId = last.topMessageId;
-			_dialogsProcess->offsetDate = last.topMessageDate;
-			_dialogsProcess->offsetPeer = last.input;
-
+		_dialogsProcess->processedCount += info.list.size();
+		const auto last = info.list.empty()
+			? Data::DialogInfo()
+			: info.list.back();
 		appendDialogsSlice(std::move(info));
 
-			const auto count = _dialogsProcess->info.list.size();
-			if (!_dialogsProcess->progress(count)) {
+		if (!_dialogsProcess->progress(_dialogsProcess->processedCount)) {
 			return;
 		}
 
-			requestDialogsSlice();
+		if (!finished && last.topMessageDate > 0) {
+			_dialogsProcess->offsetId = last.topMessageId;
+			_dialogsProcess->offsetDate = last.topMessageDate;
+			_dialogsProcess->offsetPeer = last.input;
+		} else if (--_dialogsProcess->splitIndexPlusOne > 0) {
+			_dialogsProcess->offsetId = 0;
+			_dialogsProcess->offsetDate = 0;
+			_dialogsProcess->offsetPeer = MTP_inputPeerEmpty();
+		} else {
+			finishDialogsList();
+			return;
 		}
+		requestDialogsSlice();
 	}).send();
 }
 
 void ApiWrap::appendDialogsSlice(Data::DialogsInfo &&info) {
 	Expects(_dialogsProcess != nullptr);
+	Expects(_dialogsProcess->splitIndexPlusOne <= _splits.size());
 
-	appendChatsSlice(_dialogsProcess->info, std::move(info));
+	appendChatsSlice(
+		*_dialogsProcess,
+		std::move(info),
+		_dialogsProcess->splitIndexPlusOne - 1);
 }
 
 void ApiWrap::finishDialogsList() {
@@ -822,13 +912,18 @@ void ApiWrap::requestLeftChannelsSliceGeneric(FnMut<void()> done) {
 
 void ApiWrap::appendLeftChannelsSlice(Data::DialogsInfo &&info) {
 	Expects(_leftChannelsProcess != nullptr);
+	Expects(!_splits.empty());
 
-	appendChatsSlice(_leftChannelsProcess->info, std::move(info));
+	appendChatsSlice(
+		*_leftChannelsProcess,
+		std::move(info),
+		_splits.size() - 1);
 }
 
 void ApiWrap::appendChatsSlice(
-		Data::DialogsInfo &to,
-		Data::DialogsInfo &&info) {
+		ChatsProcess &to,
+		Data::DialogsInfo &&info,
+		int splitIndex) {
 	Expects(_settings != nullptr);
 
 	const auto types = _settings->types;
@@ -837,41 +932,33 @@ void ApiWrap::appendChatsSlice(
 	) | ranges::view::filter([&](const Data::DialogInfo &info) {
 		return (types & SettingsFromDialogsType(info.type)) != 0;
 	});
-	auto &list = to.list;
-	if (list.empty()) {
-		list = filtered | ranges::to_vector;
-	} else {
+	auto &list = to.info.list;
 	list.reserve(list.size() + info.list.size());
 	for (auto &info : filtered) {
+		const auto nextIndex = list.size();
+		const auto [i, ok] = to.indexByPeer.emplace(info.peerId, nextIndex);
+		if (ok) {
 			list.push_back(std::move(info));
 		}
+		list[i->second].splits.push_back(splitIndex);
+		list[i->second].messagesCountPerSplit.push_back(0);
 	}
 }
 
-void ApiWrap::requestMessagesSlice(FnMut<bool(int count)> start) {
+void ApiWrap::requestMessagesSlice() {
 	Expects(_chatProcess != nullptr);
 
-	auto handleResult = [=, start = std::move(start)](
-			const MTPmessages_Messages &result) mutable {
+	requestChatMessages(
+		_chatProcess->info.splits[_chatProcess->localSplitIndex],
+		_chatProcess->largestIdPlusOne,
+		-kMessagesSliceLimit,
+		kMessagesSliceLimit,
+		[=](const MTPmessages_Messages &result) {
 		Expects(_chatProcess != nullptr);
 
-		const auto count = result.match(
-		[](const MTPDmessages_messages &data) {
-			return data.vmessages.v.size();
-		}, [](const MTPDmessages_messagesSlice &data) {
-			return data.vcount.v;
-		}, [](const MTPDmessages_channelMessages &data) {
-			return data.vcount.v;
-		}, [](const MTPDmessages_messagesNotModified &data) {
-			return 0;
-		});
-
 		result.match([&](const MTPDmessages_messagesNotModified &data) {
 			error("Unexpected messagesNotModified received.");
 		}, [&](const auto &data) {
-			if (start && !start(count)) {
-				return;
-			}
 			if constexpr (MTPDmessages_messages::Is<decltype(data)>()) {
 				_chatProcess->lastSlice = true;
 			}
@@ -882,9 +969,17 @@ void ApiWrap::requestMessagesSlice(FnMut<bool(int count)> start) {
 				data.vchats,
 				_chatProcess->info.relativePath));
 		});
-	};
+	});
+}
+
+void ApiWrap::requestChatMessages(
+		int splitIndex,
+		int offsetId,
+		int addOffset,
+		int limit,
+		FnMut<void(MTPmessages_Messages&&)> done) {
 	if (_chatProcess->info.onlyMyMessages) {
-		mainRequest(MTPmessages_Search(
+		splitRequest(splitIndex, MTPmessages_Search(
 			MTP_flags(MTPmessages_Search::Flag::f_from_id),
 			_chatProcess->info.input,
 			MTP_string(""), // query
@@ -892,24 +987,24 @@ void ApiWrap::requestMessagesSlice(FnMut<bool(int count)> start) {
 			MTP_inputMessagesFilterEmpty(),
 			MTP_int(0), // min_date
 			MTP_int(0), // max_date
-			MTP_int(_chatProcess->offsetId),
-			MTP_int(-kMessagesSliceLimit),
-			MTP_int(kMessagesSliceLimit),
+			MTP_int(offsetId),
+			MTP_int(addOffset),
+			MTP_int(limit),
 			MTP_int(0), // max_id
 			MTP_int(0), // min_id
 			MTP_int(0) // hash
-		)).done(std::move(handleResult)).send();
+		)).done(std::move(done)).send();
 	} else {
-		mainRequest(MTPmessages_GetHistory(
+		splitRequest(splitIndex, MTPmessages_GetHistory(
 			_chatProcess->info.input,
-			MTP_int(_chatProcess->offsetId),
+			MTP_int(offsetId),
 			MTP_int(0), // offset_date
-			MTP_int(-kMessagesSliceLimit),
-			MTP_int(kMessagesSliceLimit),
+			MTP_int(addOffset),
+			MTP_int(limit),
 			MTP_int(0), // max_id
 			MTP_int(0), // min_id
 			MTP_int(0)  // hash
-		)).done(std::move(handleResult)).send();
+		)).done(std::move(done)).send();
 	}
 }
 
@@ -957,15 +1052,21 @@ void ApiWrap::finishMessagesSlice() {
 
 	auto slice = *base::take(_chatProcess->slice);
 	if (!slice.list.empty()) {
-		_chatProcess->offsetId = slice.list.back().id + 1;
+		_chatProcess->largestIdPlusOne = slice.list.back().id + 1;
 		if (!_chatProcess->handleSlice(std::move(slice))) {
 			return;
 		}
 	}
-	if (_chatProcess->lastSlice) {
-		finishMessages();
-	} else {
+	if (_chatProcess->lastSlice
+		&& (++_chatProcess->localSplitIndex
+			< _chatProcess->info.splits.size())) {
+		_chatProcess->lastSlice = false;
+		_chatProcess->largestIdPlusOne = 1;
+	}
+	if (!_chatProcess->lastSlice) {
 		requestMessagesSlice();
+	} else {
+		finishMessages();
 	}
 }
 

Telegram/SourceFiles/export/export_api_wrap.h

@@ -93,6 +93,7 @@ private:
 	struct UserpicsProcess;
 	struct FileProcess;
 	struct FileProgress;
+	struct ChatsProcess;
 	struct LeftChannelsProcess;
 	struct DialogsProcess;
 	struct ChatProcess;
@@ -100,6 +101,7 @@ private:
 	void startMainSession(FnMut<void()> done);
 	void sendNextStartRequest();
 	void requestUserpicsCount();
+	void requestSplitRanges();
 	void requestDialogsCount();
 	void requestLeftChannelsCount();
 	void finishStartProcess();
@@ -114,6 +116,8 @@ private:
 	void finishUserpicsSlice();
 	void finishUserpics();
 
+	void validateSplits();
+
 	void requestDialogsSlice();
 	void appendDialogsSlice(Data::DialogsInfo &&info);
 	void finishDialogsList();
@@ -122,9 +126,19 @@ private:
 	void requestLeftChannelsSlice();
 	void appendLeftChannelsSlice(Data::DialogsInfo &&info);
 
-	void appendChatsSlice(Data::DialogsInfo &to, Data::DialogsInfo &&info);
+	void appendChatsSlice(
+		ChatsProcess &to,
+		Data::DialogsInfo &&info,
+		int splitIndex);
 
-	void requestMessagesSlice(FnMut<bool(int count)> start = nullptr);
+	void requestMessagesCount(int localSplitIndex);
+	void requestMessagesSlice();
+	void requestChatMessages(
+		int splitIndex,
+		int offsetId,
+		int addOffset,
+		int limit,
+		FnMut<void(MTPmessages_Messages&&)> done);
 	void loadMessagesFiles(Data::MessagesSlice &&slice);
 	void loadNextMessageFile();
 	bool loadMessageFileProgress(FileProgress value);
@@ -150,6 +164,9 @@ private:
 	template <typename Request>
 	[[nodiscard]] auto mainRequest(Request &&request);
 
+	template <typename Request>
+	[[nodiscard]] auto splitRequest(int index, Request &&request);
+
 	[[nodiscard]] auto fileRequest(
 		const Data::FileLocation &location,
 		int offset);
@@ -173,6 +190,7 @@ private:
 	std::unique_ptr<LeftChannelsProcess> _leftChannelsProcess;
 	std::unique_ptr<DialogsProcess> _dialogsProcess;
 	std::unique_ptr<ChatProcess> _chatProcess;
+	QVector<MTPMessageRange> _splits;
 
 	rpl::event_stream<RPCError> _errors;
 	rpl::event_stream<Output::Result> _ioErrors;

Telegram/SourceFiles/export/export_controller.cpp

@@ -465,7 +465,9 @@ void Controller::exportNextDialog() {
 				return false;
 			}
 			_messagesWritten = 0;
-			_messagesCount = info.messagesCount;
+			_messagesCount = ranges::accumulate(
+				info.messagesCountPerSplit,
+				0);
 			setState(stateDialogs(DownloadProgress()));
 			return true;
 		}, [=](DownloadProgress progress) {
@@ -509,7 +511,9 @@ void Controller::exportNextLeftChannel() {
 				return false;
 			}
 			_messagesWritten = 0;
-			_messagesCount = info.messagesCount;
+			_messagesCount = ranges::accumulate(
+				info.messagesCountPerSplit,
+				0);
 			setState(stateLeftChannels(DownloadProgress()));
 			return true;
 		}, [=](DownloadProgress progress) {

Telegram/SourceFiles/export/export_pch.h

@@ -21,6 +21,7 @@ https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
 
 #include <vector>
 #include <map>
+#include <set>
 #include <deque>
 #include <atomic>
 

Telegram/SourceFiles/export/export_settings.h

@@ -53,6 +53,7 @@ struct Settings {
 		GroupsMask          = PrivateGroups | PublicGroups,
 		ChannelsMask        = PrivateChannels | PublicChannels,
 		GroupsChannelsMask  = GroupsMask | ChannelsMask,
+		NonChannelChatsMask = PersonalChats | BotChats | PrivateGroups,
 		AnyChatsMask        = PersonalChats | BotChats | GroupsChannelsMask,
 	};
 	using Types = base::flags<Type>;

Telegram/SourceFiles/export/output/export_output_text.cpp

@@ -490,7 +490,7 @@ Result TextWriter::writeUserpicsSlice(const Data::UserpicsSlice &data) {
 					"Photo",
 					(userpic.image.file.relativePath.isEmpty()
 						? QByteArray("(file unavailable)")
-						: userpic.image.file.relativePath.toUtf8())
+						: FormatFilePath(userpic.image.file))
 				},
 			}));
 		}
@@ -792,7 +792,12 @@ Result TextWriter::writeChatEnd() {
 	return _chats->writeBlock(SerializeKeyValue({
 		{ "Name", NameString(_dialog.name, _dialog.type) },
 		{ "Type", TypeString(_dialog.type) },
-		{ "Messages count", Data::NumberToString(_messagesCount) },
+		{
+			(_dialog.onlyMyMessages
+				? "Outgoing messages count"
+				: "Messages count"),
+			Data::NumberToString(_messagesCount)
+		},
 		{
 			"Content",
 			(_messagesCount > 0