#! /usr/bin/python
"""Cleans output from other scripts to eliminate duplicates.

When frequently sampling data, we see that records occasionally will contain
the same timestamp (due to perf recording twice in the same second).

This removes all of the duplicate timestamps for every record. Order with
respect to timestamps is not preserved. Also, the assumption is that the log
file is a csv with the first value in each row being the time in seconds from a
standard time.

"""

import argparse

parser = argparse.ArgumentParser()
parser.add_argument('filename')
args = parser.parse_args()

my_file = open(args.filename)
output_file = open('clean2.csv', 'a')
dictionary = dict()

for line in my_file:
  new_time = int(line.split(',')[0])
  dictionary[new_time] = line

for key in dictionary.keys():
  output_file.write(dictionary[key])