AnonymousTopicBasedPubSubCommunicationforMicroblogging/lib/databaseRead.go

package lib

import (
	"bytes"
	"encoding/json"
	"sort"
)

//topicPointer and textPointer should not be exported
//mb move tweet reconstruction to dbRead from servers
type Tweet struct {
	TopicPointer string
	TextPointer  int
	Topics       []string
	Text         string
	RoundPosted  int
}

var dbR = make(map[string][]Tweet)

var archive = make(map[string][]Tweet)

//needs to be dividable by roundsBeforUpdate
var roundsBeforeArchiving = 12
var roundsBeforeArchivingInc = roundsBeforeArchiving
var bytesSaved float64

var topicList []string

var archiveTopicList []string

func NewEntries(inputTweets []Tweet, whereTo int) {
	tmpdb := dbR
	if whereTo == 1 {
		tmpdb = archive
	}
	var position int = 0
	for _, tweet := range inputTweets {
		for index := range tweet.Topics {
			//new topic
			if _, ok := tmpdb[tweet.Topics[index]]; !ok {
				if whereTo == 0 {
					topicList = append(topicList, tweet.Topics[index])
				} else {
					archiveTopicList = append(archiveTopicList, tweet.Topics[index])
				}
			}
			//new tweet
			if index == 0 {
				position = len(tmpdb[tweet.Topics[0]])
				tmpdb[tweet.Topics[index]] = append(tmpdb[tweet.Topics[0]], tweet)
			} else {
				//known tweet
				//setting pointer for all other Topics
				if whereTo == 1 {
					bytesSaved += float64(len(tweet.Topics)) + float64(len(tweet.Text))
				}
				topic := tweet.Topics[index]
				var pointerTweet Tweet
				pointerTweet.TopicPointer = tweet.Topics[0]
				pointerTweet.TextPointer = position
				pointerTweet.Topics = nil
				pointerTweet.Text = ""
				tmpdb[topic] = append(tmpdb[topic], pointerTweet)
			}
		}
	}
	if whereTo == 0 {
		dbR = tmpdb
	} else {
		archive = tmpdb
	}
}

//todo! add round to pirquery only get tweets that have been posted from that round onward
func GetTweets(pirQuery []byte, dataLength int, whereFrom int, pubKey [32]byte) []byte {
	tmpdb := dbR
	if whereFrom == 1 {
		tmpdb = archive
	}

	var wantedTopics = getNamesForTopics(pirQuery, whereFrom)
	tweetsToReturn := make([][]Tweet, len(wantedTopics))
	for index, wantedTopic := range wantedTopics {
		for _, tweet := range tmpdb[wantedTopic] {
			//new Tweet
			if tweet.Text != "" {
				tweet.RoundPosted = 0
				tweetsToReturn[index] = append(tweetsToReturn[index], tweet)
			} else {
				//"copied" tweet
				//find tweet with pointers
				tweet = tmpdb[tweet.TopicPointer][tweet.TextPointer]
				tweet.RoundPosted = 0
				tweetsToReturn[index] = append(tweetsToReturn[index], tweet)
			}
		}
	}

	return tweetsToByteArray(tweetsToReturn, whereFrom, wantedTopics)
}

func getNamesForTopics(wantedIndices []byte, whereFrom int) []string {
	var topicNames []string
	tmpTopicList := topicList
	if whereFrom == 1 {
		tmpTopicList = archiveTopicList
	}
	for index, element := range wantedIndices {
		if index == len(tmpTopicList) {
			break
		}
		if element == 1 {
			topicNames = append(topicNames, tmpTopicList[index])
		}
	}
	return topicNames
}

//transform struct to byte array for sending
func tweetsToByteArray(tweetsToReturn [][]Tweet, whereFrom int, wantedTopics []string) []byte {
	//50 is an estimated nr that works, there probably is a smarter way to do this
	minimumBlockSize := 50 * maxTweetAmount(whereFrom)

	tweetsAsBytes := make([]byte, minimumBlockSize)
	for _, block := range tweetsToReturn {
		var topicPadding []string
		var blockToAppend []byte
		for index, tweet := range block {
			for topicIndex, topic := range tweet.Topics {
				blockToAppend = append(blockToAppend, []byte(topic)...)
				blockToAppend = append(blockToAppend, ","...)

				//gets the topic used for padding
				if topicIndex > 0 && index < len(wantedTopics) && topic != wantedTopics[index] {
					topicPadding = append(topicPadding, topic)
				}
			}
			//replaces last "," with ";;" bc there is text following and not another topic
			blockToAppend = blockToAppend[:len(blockToAppend)-1]
			blockToAppend = append(blockToAppend, []byte(";;")[:]...)
			blockToAppend = append(blockToAppend, []byte(tweet.Text)...)
			blockToAppend = append(blockToAppend, []byte(";")[0])
		}

		//adds padding
		blockToAppend = append(blockToAppend, []byte(";;;")[:]...)
		remainingLength := minimumBlockSize - len(blockToAppend)

		//grouping using topic from recovered tweets
		index := len(topicPadding) - 1
		for index >= 0 && remainingLength > 0 && topicPadding[index] != "" {
			paddingTweet, err := getNextTweet(topicPadding[index], index, whereFrom)
			if err {
				break
			}
			if remainingLength < len(paddingTweet) {
				break
			}
			blockToAppend = append(blockToAppend, paddingTweet...)

			remainingLength -= len(paddingTweet)
			index--
		}

		padding := bytes.Repeat([]byte(";"), remainingLength)
		blockToAppend = append(blockToAppend, padding...)
		Xor(blockToAppend, tweetsAsBytes)

	}

	return tweetsAsBytes
}

func maxTweetAmount(whereFrom int) int {
	tmpdb := dbR
	if whereFrom == 1 {
		tmpdb = archive
	}
	var max int = 0
	for i := range tmpdb {
		nrOfTweets := len(tmpdb[i])
		if nrOfTweets > max {
			max = nrOfTweets
		}
	}
	return max
}

//gets the Tweet at index from wantedTopic for padding
func getNextTweet(wantedTopic string, index, whereFrom int) ([]byte, bool) {
	tmpdb := dbR
	if whereFrom == 1 {
		tmpdb = archive
	}

	var tweetToReturn Tweet
	for tweetIndex, tweet := range tmpdb[wantedTopic] {
		if len(tmpdb[wantedTopic]) <= index {
			return nil, true
		} else if tweetIndex < index {
			continue
		} else if tweetIndex > index {
			break
		}

		//new Tweet
		if tweet.Text != "" {
			tweet.RoundPosted = 0
			tweetToReturn = tweet
		} else {
			//"copied" tweet
			//find tweet with pointers
			tweet = tmpdb[tweet.TopicPointer][tweet.TextPointer]
			tweet.RoundPosted = 0
			tweetToReturn = tweet
		}
	}

	var tweetToReturnBytes []byte
	for _, topic := range tweetToReturn.Topics {
		tweetToReturnBytes = append(tweetToReturnBytes, []byte(topic)...)
		tweetToReturnBytes = append(tweetToReturnBytes, ","...)
	}
	//replaces last "," with ";;" bc there is text following and not another topic
	tweetToReturnBytes = tweetToReturnBytes[:len(tweetToReturnBytes)-1]
	tweetToReturnBytes = append(tweetToReturnBytes, []byte(";;")[:]...)
	tweetToReturnBytes = append(tweetToReturnBytes, []byte(tweetToReturn.Text)...)
	tweetToReturnBytes = append(tweetToReturnBytes, []byte(";")[0])

	return tweetToReturnBytes, false
}

//see func name
func GetTopicList(whereFrom int) ([]byte, int) {
	tmpTopicList := topicList
	if whereFrom == 1 {
		tmpTopicList = archiveTopicList
	}
	if (len(tmpTopicList)) == 0 {
		return nil, 0
	}
	topicByteArray := new(bytes.Buffer)
	json.NewEncoder(topicByteArray).Encode(tmpTopicList)
	return topicByteArray.Bytes(), len(tmpTopicList)
}

//iterates through full dbR and moves old tweets to archive
func CleanUpdbR(round int) {

	if roundsBeforeArchiving == -1 {
		return
	}

	if roundsBeforeArchivingInc-round == 0 {

		keys := make([]string, len(dbR))
		i := 0
		for k := range dbR {
			keys[i] = k
			i++
		}
		sort.Strings(keys)

		var tweetsToArchive []Tweet

		for _, topic := range keys {
			tweets := dbR[topic]
			for i := len(tweets) - 1; i >= 0; i-- {
				if round-roundsBeforeArchiving >= tweets[i].RoundPosted {
					//only adds the tweet to the archive when there is text
					if tweets[i].Text != "" {
						tweetsToArchive = append(tweetsToArchive, tweets[i])
					}
					//delets the tweet from the array
					row := append(tweets[:i], tweets[i+1:]...)
					tweets = row
				}
			}
			dbR[topic] = tweets
		}

		NewEntries(tweetsToArchive, 1)

		roundsBeforeArchivingInc += roundsBeforeArchiving

		//redoes the whole dbR to correct pointers
		var tweetsToMain []Tweet

		for _, topic := range keys {
			tweets := dbR[topic]
			for _, tweet := range tweets {
				if tweet.Text != "" {
					tweetsToMain = append(tweetsToMain, tweet)
				}
			}
		}

		dbR = nil
		dbR = make(map[string][]Tweet)
		topicList = nil

		NewEntries(tweetsToMain, 0)
	}
}

func GetBytesSaved() float64 {
	return bytesSaved / float64(len(archive))
}