Augamented importer to specify range and size of data to be imported. Added basic test for kmeans clusterer.

Author: mpraski

data/bus-stops.csv

@@ -19,8 +19,8 @@ type dbscanClusterer struct {
 	j          chan *rangeJob
 	m          *sync.Mutex
 	w          *sync.WaitGroup
-	p          []float64
 	r          *[]int
+	p          []float64
 
 	// visited points
 	v []bool

dbscan.go

@@ -11,4 +11,5 @@ var (
 	ErrZeroMinpts     = errors.New("MinPts cannot be 0")
 	ErrZeroWorkers    = errors.New("Number of workers cannot be less than 0")
 	ErrZeroXi         = errors.New("Xi cannot be 0")
+	ErrInvalidRange   = errors.New("Range is invalid")
 )

errors.go

@@ -2,7 +2,6 @@ package clusters
 
 import (
 	"bufio"
-	"bytes"
 	"encoding/csv"
 	"io"
 	"os"
@@ -16,30 +15,26 @@ func NewImporter() *Importer {
 	return &Importer{}
 }
 
-func (i *Importer) Import(file string) ([][]float64, error) {
-	f, err := os.Open(file)
-	if err != nil {
-		return [][]float64{}, err
+func (i *Importer) Import(file string, start, end, size int) ([][]float64, error) {
+	if start < 0 || end < 0 || start > end {
+		return [][]float64{}, ErrInvalidRange
 	}
 
-	defer f.Close()
-
-	b := bufio.NewReader(f)
-
-	c, err := i.lineCount(bufio.NewReader(b))
+	f, err := os.Open(file)
 	if err != nil {
 		return [][]float64{}, err
 	}
 
-	f.Seek(0, 0)
-	b.Reset(f)
+	defer f.Close()
 
 	var (
-		d = make([][]float64, c)
-		r = csv.NewReader(b)
-		k = 0
+		d = make([][]float64, 0, size)
+		r = csv.NewReader(bufio.NewReader(f))
+		s = end - start + 1
+		g []float64
 	)
 
+Main:
 	for {
 		record, err := r.Read()
 
@@ -49,39 +44,19 @@ func (i *Importer) Import(file string) ([][]float64, error) {
 			return [][]float64{}, err
 		}
 
-		d[k] = make([]float64, 0, len(record))
+		g = make([]float64, 0, s)
 
-		for j, _ := range record {
+		for j := start; j <= end; j++ {
 			f, err := strconv.ParseFloat(record[j], 64)
 			if err == nil {
-				d[k] = append(d[k], f)
+				g = append(g, f)
 			} else {
-				return [][]float64{}, err
+				continue Main
 			}
 		}
 
-		k++
+		d = append(d, g)
 	}
 
 	return d, nil
 }
-
-func (*Importer) lineCount(r *bufio.Reader) (int, error) {
-	var (
-		buf     = make([]byte, 32*1024)
-		count   = 0
-		lineSep = []byte{'\n'}
-	)
-
-	for {
-		c, err := r.Read(buf)
-		count += bytes.Count(buf[:c], lineSep)
-
-		switch err {
-		case io.EOF:
-			return count, nil
-		default:
-			return count, err
-		}
-	}
-}

importer.go

@@ -14,9 +14,9 @@ func TestImportedLoadDataOfCorrectLengh(t *testing.T) {
 		s = 3
 	)
 
-	d, e := i.Import(f)
+	d, e := i.Import(f, 0, 2, 3)
 	if e != nil {
-		t.Errorf("Error importing data: %e", e)
+		t.Errorf("Error importing data: %s", e.Error())
 	}
 
 	if s != len(d) {
@@ -35,13 +35,13 @@ func TestImportedLoadCorrectData(t *testing.T) {
 		}
 	)
 
-	d, e := i.Import(f)
+	d, e := i.Import(f, 0, 2, 3)
 	if e != nil {
-		t.Errorf("Error importing data: %e", e)
+		t.Errorf("Error importing data: %s", e.Error())
 	}
 
 	if !fsliceEqual(d, s) {
-		t.Error("Imported data mismatch")
+		t.Error("Imported data mismatch: %v vs %v", d, s)
 	}
 }
 
@@ -64,3 +64,17 @@ func fsliceEqual(a, b [][]float64) bool {
 
 	return true
 }
+
+func BenchmarkImport(b *testing.B) {
+	var (
+		f = "data/bus-stops.csv"
+		i = NewImporter()
+	)
+
+	b.ResetTimer()
+
+	_, e := i.Import(f, 4, 5, 15000)
+	if e != nil {
+		b.Errorf("Error importing data: %s", e.Error())
+	}
+}

importer_test.go

@@ -0,0 +1,34 @@
+package clusters
+
+import (
+	"testing"
+)
+
+func TestKmeansClusterNumerMatches(t *testing.T) {
+	const (
+		C = 8
+	)
+
+	var (
+		f = "data/bus-stops.csv"
+		i = NewImporter()
+	)
+
+	d, e := i.Import(f, 4, 5, 15000)
+	if e != nil {
+		t.Errorf("Error importing data: %s", e.Error())
+	}
+
+	c, e := KMeans(1000, C, EuclideanDistance)
+	if e != nil {
+		t.Errorf("Error initializing kmeans clusterer: %s", e.Error())
+	}
+
+	if e = c.Learn(d); e != nil {
+		t.Errorf("Error learning data: %s", e.Error())
+	}
+
+	if len(c.Sizes()) != C {
+		t.Errorf("Number of clusters does not match: %d vs %d", len(c.Sizes()), C)
+	}
+}

kmeans_test.go

@@ -30,8 +30,8 @@ type opticsClusterer struct {
 	c             chan *clusterJob
 	m             *sync.Mutex
 	w             *sync.WaitGroup
-	p             []float64
 	r             *[]int
+	p             []float64
 
 	// visited points
 	v []bool