Laboratoire de l'Informatique et du Parallélisme » XLcloud

symbols(0,0,circles=1,inches=FALSE,add=TRUE)

dev.off()

return (X)

}

funct(filename, "test_bt_cool.eps")

draw <- function(dataset,out_file){

X <- dudi.pca(dataset, center=T,scale = T, scannf=F)

postscript(out_file, width = 4.0, height = 4.0,horizontal = FALSE, onefile = FALSE, paper = "special",family = "ComputerModern", encoding = "TeXtext.enc")

keep <- 100 * X$eig/sum(X$eig)

plot(X$co[,1],X$co[,2],xlim = c(-1,1), ylim = c(-1,1), asp = 1,ylab=paste("2nd principal axis (", round(keep[2],1), "%)", sep=""),xlab=paste("1st principal axis (", round(keep[1],1), "%)", sep=""))

z <- row.names(X$co)

print(z[1])

print(X$co[1,1])

k <- 1

#while (k < length(row.names(X$co))){

x <- rnorm(5, 0, 1)

arrows(x[2] , x[4] , X$co[,1], X$co[,2], code = 2, col = 1, length=.25)

#arrows(X$co[,1], X$co[,2], y0=x[2] +1, y1=x[3] +1)

#text(X$co[,1],X$co[,2],row.names(X$co),col="navy")

text(x[2] ,x[4] ,z[1],col="navy")

k <- k + 1

#}

abline(h=0,v=0)

symbols(0,0,circles=1,inches=FALSE,add=TRUE)

dev.off()

return (X)

}

funct(filename, "test_bt_cool.eps")

draw <- function(dataset,out_file){

X <- dudi.pca(dataset, center=T,scale = T, scannf=F)

postscript(out_file, width = 4.0, height = 4.0,horizontal = FALSE, onefile = FALSE, paper = "special",family = "ComputerModern", encoding = "TeXtext.enc")

keep <- 100 * X$eig/sum(X$eig)

plot(X$co[,1],X$co[,2],xlim = c(-1,1), ylim = c(-1,1), asp = 1,ylab=paste("2nd principal axis (", round(keep[2],1), "%)", sep=""),xlab=paste("1st principal axis (", round(keep[1],1), "%)", sep=""))

z <- row.names(X$co)

print(z[1])

print(X$co[1,1])

k <- 1

while (k <= length(row.names(X$co))){

x <- rnorm(5, 0, 1)

arrows(x[2] , x[4] , X$co[k,1], X$co[k,2], code = 2, col = 1, length=.25)

#arrows(X$co[,1], X$co[,2], y0=x[2] +1, y1=x[3] +1)

#text(X$co[,1],X$co[,2],row.names(X$co),col="navy")

text(x[2] ,x[4] ,z[k],col="navy")

k <- k + 1

}

abline(h=0,v=0)

symbols(0,0,circles=1,inches=FALSE,add=TRUE)

dev.off()

return (X)

}

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

filename="/Users/ghislainlandry/AppProfile/Experiments/PhaseDetect/data.raw/data.pca/BT.txt"

funct(filename, "test_bt_cool.eps")

filename="/Users/ghislainlandry/AppProfile/Experiments/PhaseDetect/data.raw/data.pca/bakup/BT.txt"

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

filename = "/Users/ghislainlandry/AppProfile/Experiments/PhaseDetect/data.raw/data.pca/data.nas/l2.bt.txt"

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

source('~/AppProfile/Experiments/scriptPdpPca.R')

funct(filename, "test_bt_cool.eps")

?as.numeric

?mode

?ld

??ld

??ForImp

install.packages("ForImp")

install.packages("mi")

?mi

??mi

library(stat)

library(stats)

?mi

x <- rnorm(100,0,1) # N(0,1)

x

y <- rbinom(100,1,invlogit(1+2*x))

library(mi)

y <- rbinom(100,1,invlogit(1+2*x))

y[seq(1,100,10)]<-NA

dat.xy <- data.frame(x,y)

head(dat.xy)

mi.binary(y~x, data = dat.xy)

x <-rnorm(100,0,1)

y <- x+4

y <- round(y)

y[y<0] <- 0

y

y[seq(1,100,10)] <- NA

?seq

dat.xy <- data.frame(x,y)

head(dat.xy)

mi.categorical(formula = y ~ x, data = dat.xy)

z <- mi.categorical(formula = y ~ x, data = dat.xy)

head(z)

head(dat.xy)

z <- as.data.frame(mi.categorical(formula = y ~ x, data = dat.xy))

head(z)

z <- mi.categorical(formula = y ~ x, data = dat.xy)

z

?mi

library(doMc)

library(doMC)

dir()

?factors

Factors

?Factors

?as.factor

?remove

?random

?rand

??random

seed <- 3433

set.seed(seed)

?runif

runif(1)

runif(1)

sample(1:6,10,replace=F)

sample(1:20,10,replace=F)

x <- sample(1:20,10,replace=F)

x[1]

df.fits <- list()

df.fits

?complete.cases()

x <- c(0.8, 0.47, 0.51, 0.73, 0.36, 0.58, 0.57, 0.85, 0.44, 0.42)

y <- c(1.39, 0.72, 1.55, 0.48, 1.19, -1.59, 1.23, -0.65, 1.49, 0.05)

fit <- lm(y ~ x -1)

fit

x <- c(10.1, 9.2, 8.4, 9.5, 7.5)

y <- c(9.9, 9.1, 8.4, 9.3, 7.2)

mean(x+y)

mean(x)

mean(y)

x

sum(x)

sum(x)*12

x <- c(0.8, 0.47, 0.51, 0.73, 0.36, 0.58, 0.57, 0.85, 0.44, 0.42)

y <- c(1.39, 0.72, 1.55, 0.48, 1.19, -1.59, 1.23, -0.65, 1.49, 0.05)

fit <- lm(y ~ 0+x)

summary(fit)

fit <- lm(y ~ x-1)

summary(fit)

x <- -5:5

y <- c(5.12, 3.93, 2.67, 1.87, 0.52, 0.08, 0.93, 2.05, 2.54, 3.87, 4.97)

knots <- rep(0,10)

splineTerms <- sapply(knots, function(k) (x > k) * (x -k))

xMat <- cbind(1, x, splineTerms)

yhat <- predict(lm(y ~ xMat - 1))

plot(x, y, frame = FALSE, pch = 21, bg = "lightblue", cex = 2)

lines(x, yhat, col = "red", lwd = 2)

fit <- lm(y ~ xMat - 1)

library(ggplot2)

?rnorm

?runif

x <- runif(10000, 50,47000)

hist(x)

con <- url("http://www.router-switch.com/search/server/?pagesize=150")

library(XML)

html <- htmlTreeParse(con, useInternalNodes=T)

url <- "http://www.router-switch.com/search/server/?pagesize=150"

html <- htmlTreeParse(url, useInternalNodes=T)

Y <- xpathSAply(html, "//List Price:", xmlValue)

Y <- xpathSApply(html, "//List Price:", xmlValue)

Y <- xpathSApply(html, "//td[@id='List Price:']", xmlValue)

head(y)

head(Y)

Y

Y[0]

xpathSApply(html, "//title", xmlValue)

head(html)

xpathSApply(html, "//td[@id='List Price:']", xmlValue)

html

doc.text = unlist(xpathApply(html, '//p', xmlValue))

head(doc.tex)

head(doc.text)

"List" %in% doc.text

doc.text

doc.text = gsub('\\n', ' ', doc.text)

doc.text

doc.text = gsub('\\r', '', doc.text)

doc.text

"Our" %in% doc.text

doc.text[338]

doc.text[339]

"Our" %in% doc.textdoc.text[339]

"Our" %in% doc.text[339]

class(doc.text[339])

?gsub

y <- gsub("  ", " ",doc.text)

y

y <- paste(doc.text, collapse=" ")

y

?grepl

y <- grepl("Our",doc.text[339])

y

doc.text

y <- grepl("Our",doc.text[303])

y

y <- grepl("List",doc.text[303])

y

y <- grepl(c("List","Our"), doc.text[303])

y <- grepl("List, Our", doc.text[303])

y

y <- grepl("List" "Our", doc.text[303])

y <- grepl("List", "Our", doc.text[303])

y

y <- grepl("$",, doc.text[303])

y <- grepl("\\$",, doc.text[303])

y <- grepl("\$",, doc.text[303])

y <- grepl("\$",doc.text[303])

y <- grepl("\\$",doc.text[303])

y

x <- rapply(lapply(doc.text, function(x){grepl("\\$",doc.text[303])}), function(z){z})

x[1:20]

FALSE %in% x

x <- rapply(lapply(doc.text, function(x){grepl("\\$",doc.text)}), function(z){z})

x[1:20]

FALSE %in% x

y <- doc.text[x]

length(y)

head(y)

class(y)

y[1]

y[2]

y <- gsub("List Price:                  USD$", "", doc.text[x])

head(y)

y <- gsub("List Price:                  USD$", "", y[1])

y

y[1]

y

gsub("List Price:                  USD\\$", "", y)

as.numeric("3,495.00")

gsub(",", "","3,495.00")

y <- doc.text[x]

head(y)

lprice <- rapply(lapply(y, function(a){gsub("List Price:                  USD\\$", "", a)}), function(z){z})

head(lprice)

lprice <- rapply(lapply(lprice, function(a){gsub("Our Price:                  USD\\$", "", a)}), function(z){z})

head(lprice)

prices <- rapply(lapply(lprice, function(x){gsub(",", "",x)}), function(z){as.numeric(z)})

prices

tail(lprice)

tail(y)

y <- doc.text[x]

tail(y)

head(y)

?ifelse

Z <- ifelse(is.na(y), True, False)

Z <- ifelse(is.na(y), T, F)

lean(Z)

length(Z)

l <- y[!Z]

l

serversPrices <- function(pagelink){

# Read and parse HTML file

html <- htmlTreeParse(pagelink, useInternalNodes=T)

# Extract all the paragraphs (HTML tag is p, starting at

# the root of the document). Unlist flattens the list to

# create a character vector.

doc.text = unlist(xpathApply(html, '//p', xmlValue))

# Replace all \n by spaces

doc.text = gsub('\\n', ' ', doc.text)

doc.text = gsub('\\r', ' ', doc.text)

x <- rapply(lapply(doc.text, function(x){grepl("\\$",doc.text)}), function(z){z})

y <- doc.text[x]

y <- ifelse(is.na(y), True, False)

lprice <- rapply(lapply(y, function(a){gsub("List Price:                  USD\\$", "", a)}),

function(z){z})

lprice <- rapply(lapply(lprice, function(a){gsub("Our Price:                  USD\\$", "", a)}),

function(z){z})

prices <- rapply(lapply(lprice, function(x){gsub(",", "",x)}),

function(z){as.numeric(z)})

prices

}

prices <- serversPrices("http://www.router-switch.com/search/server/?pagesize=150")

serversPrices <- function(pagelink){

# Read and parse HTML file

html <- htmlTreeParse(pagelink, useInternalNodes=T)

# Extract all the paragraphs (HTML tag is p, starting at

# the root of the document). Unlist flattens the list to

# create a character vector.

doc.text = unlist(xpathApply(html, '//p', xmlValue))

# Replace all \n by spaces

doc.text = gsub('\\n', ' ', doc.text)

doc.text = gsub('\\r', ' ', doc.text)

x <- rapply(lapply(doc.text, function(x){grepl("\\$",doc.text)}), function(z){z})

y <- doc.text[x]

y <- ifelse(is.na(y), T, F)

lprice <- rapply(lapply(y, function(a){gsub("List Price:                  USD\\$", "", a)}),

function(z){z})

lprice <- rapply(lapply(lprice, function(a){gsub("Our Price:                  USD\\$", "", a)}),

function(z){z})

prices <- rapply(lapply(lprice, function(x){gsub(",", "",x)}),

function(z){as.numeric(z)})

prices

}

prices <- serversPrices("http://www.router-switch.com/search/server/?pagesize=150")

warnings()

length(prices)

serversPrices <- function(pagelink){

# Read and parse HTML file

html <- htmlTreeParse(pagelink, useInternalNodes=T)

# Extract all the paragraphs (HTML tag is p, starting at

# the root of the document). Unlist flattens the list to

# create a character vector.

doc.text = unlist(xpathApply(html, '//p', xmlValue))

# Replace all \n by spaces

doc.text = gsub('\\n', ' ', doc.text)

doc.text = gsub('\\r', ' ', doc.text)

x <- rapply(lapply(doc.text, function(x){grepl("\\$",doc.text)}), function(z){z})

y <- doc.text[x]

#y <- ifelse(is.na(y), T, F)

lprice <- rapply(lapply(y, function(a){gsub("List Price:                  USD\\$", "", a)}),

function(z){z})

lprice <- rapply(lapply(lprice, function(a){gsub("Our Price:                  USD\\$", "", a)}),

function(z){z})

Z <- ifelse(is.na(y), T, F)

lprice <- lprice[!Z]

prices <- rapply(lapply(lprice, function(x){gsub(",", "",x)}),

function(z){as.numeric(z)})

prices

}

prices <- serversPrices("http://www.router-switch.com/search/server/?pagesize=150")

serversPrices <- function(pagelink){

# Read and parse HTML file

html <- htmlTreeParse(pagelink, useInternalNodes=T)

# Extract all the paragraphs (HTML tag is p, starting at

# the root of the document). Unlist flattens the list to

# create a character vector.

doc.text = unlist(xpathApply(html, '//p', xmlValue))

# Replace all \n by spaces

doc.text = gsub('\\n', ' ', doc.text)

doc.text = gsub('\\r', ' ', doc.text)

x <- rapply(lapply(doc.text, function(x){grepl("\\$",doc.text)}), function(z){z})

y <- doc.text[x]

#y <- ifelse(is.na(y), T, F)

lprice <- rapply(lapply(y, function(a){gsub("List Price:                  USD\\$", "", a)}),

function(z){z})

lprice <- rapply(lapply(lprice, function(a){gsub("Our Price:                  USD\\$", "", a)}),

function(z){z})

Z <- ifelse(is.na(lprice), T, F)

lprice <- lprice[!Z]

prices <- rapply(lapply(lprice, function(x){gsub(",", "",x)}),

function(z){as.numeric(z)})

prices

}

prices <- serversPrices("http://www.router-switch.com/search/server/?pagesize=150")

length(prices)

prices

serversPrices <- function(pagelink){

# Read and parse HTML file

html <- htmlTreeParse(pagelink, useInternalNodes=T)

# Extract all the paragraphs (HTML tag is p, starting at

# the root of the document). Unlist flattens the list to

# create a character vector.

doc.text = unlist(xpathApply(html, '//p', xmlValue))

# Replace all \n by spaces

doc.text = gsub('\\n', ' ', doc.text)

doc.text = gsub('\\r', ' ', doc.text)

x <- rapply(lapply(doc.text, function(x){grepl("\\$",doc.text)}), function(z){z})

y <- doc.text[x]

#y <- ifelse(is.na(y), T, F)

lprice <- rapply(lapply(y, function(a){gsub("List Price:                  USD\\$", "", a)}),

function(z){z})

lprice <- rapply(lapply(lprice, function(a){gsub("Our Price:                  USD\\$", "", a)}),

function(z){z})

Z <- ifelse(is.na(lprice), T, F)

lprice <- lprice[!Z]

lprice

#prices <- rapply(lapply(lprice, function(x){gsub(",", "",x)}),

#                 function(z){as.numeric(z)})

#prices

}

prices <- serversPrices("http://www.router-switch.com/search/server/?pagesize=150")

prices

lprice <- rapply(lapply(lprice, function(a){gsub("Our Price:                  USD\\$", "", a)}),

function(z){z})

lprice

doc.text = unlist(xpathApply(html, '//p', xmlValue))

doc.text = gsub('\\n', ' ', doc.text)

doc.text = gsub('\\r', ' ', doc.text)

doc.text

x <- rapply(lapply(doc.text, function(x){grepl("Price",doc.text)}), function(z){z})

head(x)

y <- doc.text[x]

length(y)

head(y)

tail(y)

Z <- ifelse(is.na(y), T, F)

y <- y[!Z]

tail(y)

t <- y

length(y)

head(y)

gsub(" $", "", "Our Price:                   USD$4,598.00                                      ")

gsub(" $ ", "", "Our Price:                   USD$4,598.00                                      ")

gsub("\\s", " ", "Our Price:                   USD$4,598.00                                      ")

gsub("\t", " ", "Our Price:                   USD$4,598.00                                      ")

gsub("\\t", "", "Our Price:                   USD$4,598.00                                      ")

gsub("\\t", "", "Our Price:                   USD$4,598.00                  ")

str.replace(/[\t\n\r]/gm,'')

str <- "Our Price:                   USD$4,598.00                  "

str.replace(/[\t\n\r]/gm,'')

y[72]

gsub("\t","",y[72])

gsub("\t","",as.character(y[72]))

gsub(" ","",as.character(y[72]))

lprice <- rapply(lapply(y, function(a){" ","",as.character(a)}),

function(z){z})

lprice <- rapply(lapply(y, function(a){" ","",as.character(a)}),function(z){z})

lprice <- rapply(lapply(y, function(a){gsub(" ","",as.character(a))}),function(z){z})

head(lprice)

lprice <- rapply(lapply(y, function(a){gsub("List Price:USD\\$", "", a)}),

function(z){z})

head(lprice)

lprice <- rapply(lapply(y, function(a){gsub(" ","",as.character(a))}),function(z){z})

lprice <- rapply(lapply(lprice, function(a){gsub("List Price:USD\\$", "", a)}),

function(z){z})

head(lprice)

lprice <- rapply(lapply(lprice, function(a){gsub("ListPrice:USD\\$", "", a)}),

function(z){z})

head(lprice)

lprice <- rapply(lapply(lprice, function(a){gsub("OurPrice:USD\\$", "", a)}),

function(z){z})

head(lprice)

prices <- rapply(lapply(lprice, function(x){gsub(",", "",x)}),

function(z){as.numeric(z)})

prices

hist(prices)

?rweibull

hist(rweibull(1e5,1.5,33))

mean(prices)

hist(rweibull(1e5,1.5,mean(prices)))

?rweibull

y <- rweibull(1e5,1.5,mean(prices))

mean(y)

y <- rweibull(1e5,2,mean(prices))

mean(y)

y <- rweibull(1e5,1,mean(prices))

mean(y)

hist(y)

y <- rweibull(1e5,1.5,mean(prices))

hist(y)

y <- rweibull(1e5,3.5,mean(prices))

hist(y)

y <- rweibull(1e5,2.5,mean(prices))

hist(y)

y <- rweibull(1e5,2,mean(prices))

hist(y)

mean(y)

setwd("/Users/ghislainlandry/Enseignements/HaasAnalysis/Cat1")

dir()

require(xlsReadWrite)

require(XLConnect)

require(XLConnect)

require(xlsx)

?read.xlsx

conf1_df <- read.xlsx("categoryOneServer.xlsx", sheetName = "conf1")

head(conf1_df)

procs <- read.xlsx("conf1extensions.xlsx", sheetName = "procs")

head(procs)

sep(nrow(procs))

seq(nrow(procs))

procs$id <- seq(nrow(procs))

head(procs)

y <- merge(procs, conf1_df, by= NULL)

head(y)

#!/usr/bin/env python

from __future__ import division

import numpy as np

import pandas as pd

from numpy import genfromtxt

from pandas import DataFrame

import re

import math

import sys

import os

def duplicate(x, n):

	return [x[0]] *n

basedirs = ["Cat1", "Cat2", "Cat3"]

for directory in basedirs:

require(xlsx)

conf1_df <- read.xlsx("categoryOneServer.xlsx", sheetName = "conf1")

conf1_df$id <- seq(nrow(conf1_df))

procs <- read.xlsx("conf1extensions.xlsx", sheetName = "procs")

procs$id <- seq(nrow(procs))

#!/usr/bin/env python

from __future__ import division

import numpy as np

import pandas as pd

from numpy import genfromtxt

from pandas import DataFrame

import re

import math

import sys

def duplicate(x, n):

	return [x[0]] *n

cat1 = pd.ExcelFile("categoryOneServer.xlsx")

dico = {}

configurations = ["conf1", "conf2"]

for item in configurations:

	df_conf1 = cat1.parse("conf1")

	df_conf1["id"] = range(1, df_conf1.shape[0] + 1)

	extension_file = item + "extensions.xlsx"

	ext = pd.ExcelFile(extension_file)

	procs = ext.parse("procs")

	procs["id"] = range(1, procs.shape[0] + 1)

	mem = ext.parse("mem")

	mem["id"] = range(1, mem.shape[0] + 1)

	storage = ext.parse("storage3inch")

	storage["id"] = range(1, storage.shape[0] + 1)

	#network = ext.parse("network")

	#network["id"] = range(1, network.shape[0] + 1)

	df1 = df_conf1

	df1["procUnitCost"] = 0

	df = df1.merge(procs, how='outer', copy=False)

	colnames = list(df.columns)

	proc_names = list(procs.columns)

	colnames_t = [x for x in colnames if x not in proc_names]

	colnames = [x for x in colnames_t if x not in ["id"]]

	df[colnames] = df[colnames].apply(duplicate, 0, args=[df.shape[0]])

	data = df

	data["MemUnitCost"] = 0

	for i in range(df.shape[0]):

		d = pd.DataFrame(df.iloc[i, ]).T # get the transpose of the original data frame

		d1 = d.merge(mem, how="outer", copy=False)

		colnames = list(d1.columns)

		mem_names = list(mem.columns)

		colnames_t = [x for x in colnames if x not in mem_names]

		colnames = [x for x in colnames_t if x not in ["id"]]

		d1[colnames] = d1[colnames].apply(duplicate, 0, args=[d1.shape[0]])

		data = data.append(d1)

	## process storage

	df = data

	data["StorageUnitCost"] = 0

	for i in range(df.shape[0]):

		d = pd.DataFrame(df.iloc[i, ]).T # get the transpose of the original data frame

		d1 = d.merge(storage, how="outer", copy=False)

		colnames = list(d1.columns)

		storage_names = list(storage.columns)

		colnames_t = [x for x in colnames if x not in storage_names]

		colnames = [x for x in colnames_t if x not in ["id"]]

		d1[colnames] = d1[colnames].apply(duplicate, 0, args=[d1.shape[0]])

		data = data.append(d1)

	dico[item] = data

df = dico["conf1"]

## remove conf1 from the list

configurations.remove("conf1")

for items in configurations:

	df.append(dico[items])

#df = dico["conf1"].append(dico['conf2'])

print df.head()

df["category"] = "standard"

cost_fileds = [v for v in df.columns if "Cost" in v]

df["Cost"] = 0

for items in cost_fileds:

	df["Cost"] += df[items]

df.to_excel("standard.xlsx", sheet_name='cat', engine='xlsxwriter',index=False)

#!/usr/bin/env python

from __future__ import division

import numpy as np

import pandas as pd

from numpy import genfromtxt

from pandas import DataFrame

import re

import math

import sys

def duplicate(x, n):

	return [x[0]] *n

cat1 = pd.ExcelFile("categoryOneServer.xlsx")

dico = {}

configurations = ["conf1", "conf2"]

for item in configurations:

	df_conf1 = cat1.parse("conf1")

	df_conf1["id"] = range(1, df_conf1.shape[0] + 1)

	extension_file = item + "extensions.xlsx"

	ext = pd.ExcelFile(extension_file)

	procs = ext.parse("procs")

	procs["id"] = range(1, procs.shape[0] + 1)

	mem = ext.parse("mem")

	mem["id"] = range(1, mem.shape[0] + 1)

	storage = ext.parse("storage3inch")

	storage["id"] = range(1, storage.shape[0] + 1)

	#network = ext.parse("network")

	#network["id"] = range(1, network.shape[0] + 1)

	df1 = df_conf1

	df1["procUnitCost"] = 0

	df = df1.merge(procs, how='outer', copy=False)

	colnames = list(df.columns)

	proc_names = list(procs.columns)

	colnames_t = [x for x in colnames if x not in proc_names]

	colnames = [x for x in colnames_t if x not in ["id"]]

	df[colnames] = df[colnames].apply(duplicate, 0, args=[df.shape[0]])

	data = df

	data["MemUnitCost"] = 0

	for i in range(df.shape[0]):

		d = pd.DataFrame(df.iloc[i, ]).T # get the transpose of the original data frame

		d1 = d.merge(mem, how="outer", copy=False)

		colnames = list(d1.columns)

		mem_names = list(mem.columns)

		colnames_t = [x for x in colnames if x not in mem_names]

		colnames = [x for x in colnames_t if x not in ["id"]]

		d1[colnames] = d1[colnames].apply(duplicate, 0, args=[d1.shape[0]])

		data = data.append(d1)

	## process storage

	df = data

	data["StorageUnitCost"] = 0

	for i in range(df.shape[0]):

		d = pd.DataFrame(df.iloc[i, ]).T # get the transpose of the original data frame

		d1 = d.merge(storage, how="outer", copy=False)

		colnames = list(d1.columns)

		storage_names = list(storage.columns)

		colnames_t = [x for x in colnames if x not in storage_names]

		colnames = [x for x in colnames_t if x not in ["id"]]

		d1[colnames] = d1[colnames].apply(duplicate, 0, args=[d1.shape[0]])

		data = data.append(d1)

	dico[item] = data

df = dico["conf1"]

## remove conf1 from the list

configurations.remove("conf1")

for items in configurations:

	df.append(dico[items])

#df = dico["conf1"].append(dico['conf2'])

print df.head()

df["category"] = "standard"

cost_fileds = [v for v in df.columns if "Cost" in v]

df["Cost"] = 0

for items in cost_fileds:

	df["Cost"] += df[items]

df.to_excel("standard.xlsx", sheet_name='cat', engine='xlsxwriter',index=False)

#!/usr/bin/env python

from __future__ import division

import numpy as np

import pandas as pd

from numpy import genfromtxt

from pandas import DataFrame

import re

import math

import sys

import os

from optparse import OptionParser

import logging

'''

python version: 

2.7.8 |Anaconda 2.0.1 (x86_64)

full description:

2.7.8 |Anaconda 2.0.1 (x86_64)| (default, Aug 21 2014, 15:21:46) \n[GCC 4.2.1 (Apple Inc. build 5577)]

'''

np.random.seed(12345)

class GenerateDate():

	def __init__(self, nber_servers, 

		maintainanceCost, 

		maintainance_hours,

		software_init_cost, 

		software_update_cost,

		network_hardware_cost,

		installation_cost,

		one_time_cost,

		platform_cost_hour,

		monthly_cost_instance,

		cost_extra_gb_memory,

		cost_extra_gb_storage,

		number_years,

		number_cores,

		memory_size,

		storage_size,

		storage_type,

		usage_hours):

		self.number_servers = nber_servers

		self.basedirs = ["Cat1", "Cat2"]

		self.dfservers = pd.DataFrame()

		self.maintainanceCost = maintainanceCost

		self.maintainance_hours = maintainance_hours

		self.software_init_cost = software_init_cost

		self.software_update_cost = software_update_cost

		self.network_hardware_cost= network_hardware_cost

		self.installation_cost = installation_cost

		self.one_time_cost = one_time_cost

		self.platform_cost_hour = platform_cost_hour

		self.monthly_cost_instance = monthly_cost_instance

		self.cost_extra_gb_memory = cost_extra_gb_memory #cost per extra giga bite of memory

		self.cost_extra_gb_storage = cost_extra_gb_storage

		self.number_years = number_years

		self.number_cores = number_cores

		self.memory_size = memory_size

		self.storage_size = storage_size

		self.storage_type = storage_type

		self.usage_hours = usage_hours

	def duplicate(self, x, n):

		return [x[0]] *n

	def getItem(self, item):

		if item == "procs":

			return "procUnitCost"

		else:

			if item == "mem":

				return "MemUnitCost"

			else:

				if item == "storage3inch" or item == "storage2inch":

					return "StorageUnitCost"

				else:

					if item == "network":

						return "netUnitCost"

	def difference_percentage(self, x):

		if np.nan in x:

			return np.nan

		else:

			return round((x[0] / x[1]) * 100, 2)

	'''

		base configuration data frame

		file name for the corresponding extension file

		flag to identify the appropriate component

	'''

	def createServerSet(self, baseConfDf, extensionfile):

		xcl_file = pd.ExcelFile(extensionfile)

		dframes = {sheet_name: xcl_file.parse(sheet_name) for sheet_name in xcl_file.sheet_names}

		#

		keys = dframes.keys()

		## add the processor unit cost

		baseConfDf[self.getItem("procs")] = 0

		#

		data = baseConfDf.merge(dframes["procs"], how="outer", copy=False)

		colnames = list(data.columns)

		proc_names = list(dframes["procs"])

		colnames_t = [x for x in colnames if x not in proc_names]

		colnames = [x for x in colnames_t if x not in ["id"]]

		data[colnames] = data[colnames].apply(self.duplicate, 0, args=[data.shape[0]])		

		ext = [v for v in keys if v in ["mem", "network","storage3inch", "storage2inch"]]

		#print ext

		if ("storage3inch" in ext) and ("storage2inch" in ext):

			ext.remove("storage3inch")

			ext.remove("storage2inch")

			for elt in ext:

				#df = data

				data[self.getItem(elt)] = 0

				df = data

				for i in range(df.shape[0]):

					d = pd.DataFrame(df.iloc[i, ]).T # get the transpose of the original data frame

					d1 = d.merge(dframes[elt], how="outer", copy=False)

					colnames = list(d1.columns)

					names = list(dframes[elt].columns)

					colnames_t = [x for x in colnames if x not in names]

					colnames = [x for x in colnames_t if x not in ["id"]] ## to be removed

					d1[colnames] = d1[colnames].apply(self.duplicate, 0, args=[d1.shape[0]])

					data = data.append(d1)

			for item in ["procUnitCost", "MemUnitCost", "StorageUnitCost", "netUnitCost"]:

				if item not in list(data):

					data[item] = 0	

			threeinch = data.copy()

			twoinche = data.copy()

			#df = threeinch

			threeinch[self.getItem("storage3inch")] = 0

			df = threeinch

			for i in range(df.shape[0]):

				d = pd.DataFrame(df.iloc[i, ]).T 

				d1 = d.merge(dframes["storage3inch"], how="outer", copy=False)

				colnames = list(d1.columns)

				names = list(dframes["storage3inch"].columns)

				colnames_t = [x for x in colnames if x not in names]

				colnames = [x for x in colnames_t if x not in ["id"]] ## to be removed

				d1[colnames] = d1[colnames].apply(self.duplicate, 0, args=[d1.shape[0]])

				threeinch = threeinch.append(d1)

			data = data.append(threeinch)

			del(threeinch)

			#df = twoinche

			twoinche[self.getItem("storage2inch")] = 0

			df = twoinche

			for i in range(df.shape[0]):

				d = pd.DataFrame(df.iloc[i, ]).T 

				d1 = d.merge(dframes["storage2inch"], how="outer", copy=False)

				colnames = list(d1.columns)

				names = list(dframes["storage2inch"].columns)

				colnames_t = [x for x in colnames if x not in names]

				colnames = [x for x in colnames_t if x not in ["id"]] ## to be removed

				d1[colnames] = d1[colnames].apply(self.duplicate, 0, args=[d1.shape[0]])

				twoinche = twoinche.append(d1)

			data = data.append(twoinche)	

			del(twoinche)

		else:

			for elt in ext:

				#df = data

				data[self.getItem(elt)] = 0

				df = data

				for i in range(df.shape[0]):

					d = pd.DataFrame(df.iloc[i, ]).T # get the transpose of the original data frame

					d1 = d.merge(dframes[elt], how="outer", copy=False)

					colnames = list(d1.columns)

					names = list(dframes[elt].columns)

					colnames_t = [x for x in colnames if x not in names]

					colnames = [x for x in colnames_t if x not in ["id"]] ## to be removed

					d1[colnames] = d1[colnames].apply(self.duplicate, 0, args=[d1.shape[0]])

					data = data.append(d1)

		for item in ["procUnitCost", "MemUnitCost", "StorageUnitCost", "netUnitCost"]:

			if item not in list(data):

				data[item] = 0

		return data

	def merge_datasets(self):

		for directory in self.basedirs:

			APP_ROOT = os.path.dirname(os.path.abspath(" "))

			#print APP_ROOT

			path = os.path.join(APP_ROOT, directory)

			#print path

			baseconfigs = [ v for v in os.listdir(path) if "category" in v and "$" not in v]

			#print directory

			dataFrames = pd.DataFrame()

			extensions = [ v for v in os.listdir(path) if "extensions" in v and "$" not in v]

			#print ' '.join(extensions)

			for base in baseconfigs:

				filename = path +"/"+base

				xl_file = pd.ExcelFile(filename)

				## create a dictionary containing a DataFrame for every sheet

				dfs = {sheet_name: xl_file.parse(sheet_name, na_value="na") 

					for sheet_name in xl_file.sheet_names}

				dataframe = pd.DataFrame()

				for items in dfs.keys():

					extensionFilename = [v for v in extensions if items in v]

					filename = path +"/"+extensionFilename[0]

					#print filename

					dframes = self.createServerSet(dfs[items], filename)

					dataframe = dataframe.append(dframes)

			#for col in ["procUnitCost", "MemUnitCost", "StorageUnitCost", "netUnitCost"]:

			#	if col not in list(dataframe.columns):

			#		dataframe[col] = 0	

				dataFrames = dataFrames.append(dataframe)

			dataFrames["category"] = directory

			self.dfservers = self.dfservers.append(dataFrames)

		cost_fileds = [v for v in self.dfservers.columns if "Cost" in v]

		self.dfservers["Cost"] = 0

		for item_ in cost_fileds:

			self.dfservers["Cost"] += map(lambda y: float(y) if y != "na" else np.nan, 

				self.dfservers[item_]) #dfservers[items]

		self.dfservers.to_excel("dataset.xlsx", sheet_name='cat',

			engine='xlsxwriter',index=False)

	def filter_dataframe(self,items):

		if items == "numberCores":

			self.dfservers["subset"] = map(lambda x: 

					True if int(x) >= self.number_cores else False,

					self.dfservers[items])

			df = self.dfservers[self.dfservers["subset"] == True]

			self.dfservers = df

		else:

			if items == "storageType":

				self.dfservers["subset"] = map(lambda x:

						True if str(x) == self.storage_type else False, 

						self.dfservers[items])

				df = self.dfservers[self.dfservers["subset"] == True]

				self.dfservers = df

			else:

				if items == "memory":

					self.dfservers["subset"] = map(lambda x:

							True if int(x) >= self.memory_size else False, 

							self.dfservers[items])

					df = self.dfservers[self.dfservers["subset"] == True]

					self.dfservers = df

				else:

					if items == "storageSize":

						self.dfservers["subset"] = map(lambda x:

								True if int(x) >= self.storage_size else False, 

								self.dfservers[items])

						df = self.dfservers[self.dfservers["subset"] == True]

						self.dfservers = df

	def process_data(self):

		"""

		The algorithm keeps a constant number of maintainance hours over the 

		years

		Software cost is relatively low but can be is provided as a parameter 

		to the program.

		We only use servers with the following characteristics 

		-- number of core == 4

		-- memory 48 GB

		-- SSD storage greater than SSD 160GB

		"""

		filter_list = ["numberCores", "storageType", "storageSize", "memory"]

		for elt in filter_list:

			self.filter_dataframe(elt)

		del(self.dfservers["subset"])

		self.dfservers["extra_memory_cost"] = (self.dfservers["memory"] 

			- self.memory_size)*self.cost_extra_gb_memory

		self.dfservers["extra_storage_cost"] = (self.dfservers["storageSize"] 

			- self.storage_size)*self.cost_extra_gb_storage

		#note that the model assumes that softeware are updated every year 

		# even at cost 0, this cost include licences and so on.

		if (self.number_years >= 3):

			maintainance_hours_year = self.maintainance_hours * 12 

			## Compute the cost of a home made data center 

			self.dfservers["traditional_hpc_cost"] = self.dfservers["Cost"] * self.number_servers + \

						(self.number_years * maintainance_hours_year * self.maintainance_hours)  + \

						self.software_init_cost + \

						self.number_years * self.software_update_cost + \

						self.network_hardware_cost + \

						self.installation_cost

			# Compute the cost of corresponding server instances rented, 

			# remember that the base period is 3 years

			# We will call this cost the outsource cost 

			self.dfservers["ondemand_hpc_cost"] = self.number_servers * ( \

							self.one_time_cost + \

							self.platform_cost_hour * self.usage_hours * 365 * self.number_years) +\

							(self.number_years * maintainance_hours_year *\

							self.maintainance_hours) +\

							self.number_years * self.software_update_cost + \

							self.software_init_cost

			self.dfservers["ondemand_hpc_cost"] = self.dfservers["ondemand_hpc_cost"] +\

								self.number_servers * (self.dfservers["extra_memory_cost"] +\

								self.dfservers["extra_storage_cost"])

			self.dfservers["Cost difference"] = \

							self.dfservers["ondemand_hpc_cost"] - \

							self.dfservers["traditional_hpc_cost"]

			tmp_df = self.dfservers[["Cost difference", "ondemand_hpc_cost"]]

			self.dfservers["Cost difference (%)"] = map(lambda x: round(x, 2), 

					tmp_df.apply(self.difference_percentage, 1))

			self.dfservers["Cost difference (HPC ondemand - traditional)"] = \

							self.dfservers["Cost difference"]

			del(self.dfservers["Cost difference"])

		else:

			try :

				raise Exception

			except xception as inst:

				print("The number of years should be at leat 3! program exits on %s", inst)

		""" Update the server data set with new parameters """

		self.dfservers.to_excel("dataset_analysis.xlsx", sheet_name='cat',

			engine='xlsxwriter',index=False)

def main(argv):

	"""Main entrypoint."""

	## Data used here are for the UK franckfort Amazon region

    	try:

        # Parse command-line options

        	parser = OptionParser()

        	parser.add_option("-n", "--servers", dest="servers", 

        		metavar="NUMBER_SERVRES", action="store", type="int", default=1000,

        		help="number of servers for each observation")

       		parser.add_option("--hour_maintainance", dest="maintainance_hours", 

        		metavar="MAINTAINANCE_HOURS", action="store", type="int", default=15,

        		help="number of hours of maintainance per month")

       		parser.add_option("--maintananceCost", dest="maintainance_cost", 

        		metavar="MAINTAINANCE_COST", action="store", type="float", default=50,

        		help="cost per maintainance hour")

        	parser.add_option("--software_init_cost", dest="software_cost", 

        		metavar="SOFTWARE_INITIAL_COST", action="store", type="float", default=5000,

        		help="initial software cost (this include network software)")

        	parser.add_option("--software_update_cost", dest="software_update", 

        		metavar="SOFTWARE_UPDATE", action="store", type="float", default=10000,

        		help="cost associate to software update including licences")

        	parser.add_option("--network_hardware_cost", dest="network_hardware", 

        		metavar="NETWORK_HARDWARE", action="store", type="float", default=15000,

        		help="initial cost associated to network equipments purchase")

        	parser.add_option("--installation_cost", dest="install_cost", 

        		metavar="INTALLATION_COST", action="store", type="float", default=10000,

        		help="initial cost associated to network equipments purchase")

        	## One time cost for a period of 3 years

        	parser.add_option("--one_time_cost", dest="one_time_cost", 

        		metavar="ONE_TIME_COST", action="store", type="int", default=2690.14,

        		help="one time set up cost when you book instances from amazone")

        	parser.add_option("--platform_cost_hour", dest="cost_hour", 

        		metavar="PLATFORM_COST_HOUR", action="store", type="float", default= 0.543,

        		help="one time set up cost when you book instances from amazone") 

		parser.add_option("--monthly_cost_instance", dest="cost_monthly", 

        		metavar="COST_HOUR", action="store", type="float", default=108.64,

        		help="monthly cost of a single machine book instances from amazone") 

        	parser.add_option("--cost_extra_gb_memory", dest="extra_memory", 

        		metavar="EXTRA_MEMORY", action="store", type="float", default=1.23,

        		help="cost per hour per giga byte of extra memory") 

        	parser.add_option("--cost_extra_gb_storage", dest="extra_storage", 

        		metavar="EXTRA_STORAGE", action="store", type="float", default=0.82,

        		help="cost per hour per giga byte of extra storage") 

        	parser.add_option("--number_years", dest="number_years", 

        		metavar="NUMBER_YEARS", action="store", type="int", default=3,

        		help="Number of years over which you want to predict") 

        	parser.add_option("--number_cores", dest="number_cores", 

        		metavar="NUMBER_CPU_CORES", action="store", type="int", default=8,

        		help="Number of CPU cores per host")

        	parser.add_option("--memory_size", dest="memory_size", 

        		metavar="MEMORY_SIZE", action="store", type="int", default=48,

        		help="Memory size expected, max is 768")

        	parser.add_option("--storage_size", dest="storage_size", 

        		metavar="STORAGE_SIZE", action="store", type="int", default=48,

        		help="Storage size expected")        

        	parser.add_option("--storage_type", dest="storage_type", 

        		metavar="STORAGE_TYPE", action="store", type="string", default="ssd sata",

        		help="Storage type expected")

        	parser.add_option("--usage_ours", dest="usage_hours", 

        		metavar="STORAGE_TYPE", action="store", type="int", default=24,

        		help="number of hours the system is used per day")

       		## baseline: c3.2xlarge	8	28	15	2 x 80 SSD	$0.420 par heure

       		## upfront	monthly		hourly

       		## $3288 	$132.86 	$0.3071 	

       		options, args = parser.parse_args(argv[1:])

		gen = GenerateDate(options.servers,

			options.maintainance_hours, 

			options.maintainance_cost, 

			options.software_cost,

			options.software_update,

			options.network_hardware,

			options.install_cost,

			options.one_time_cost,

			options.cost_hour,

			options.cost_monthly,

			options.extra_memory,

			options.extra_storage,

			options.number_years,

			options.number_cores,

			options.memory_size,

			options.storage_size,

			options.storage_type,

			options.usage_hours)

 		gen.merge_datasets()

 		gen.process_data()

    	except Exception as e:

    		print "Exception % rised" % e

    		return 1

    	return 0

if __name__ == "__main__":

    sys.exit(main(sys.argv))

#!/usr/bin/env python

from __future__ import division

import numpy as np

import pandas as pd

from numpy import genfromtxt

from pandas import DataFrame

import re

import math

import sys

import os

from optparse import OptionParser

import logging

'''

python version: 

2.7.8 |Anaconda 2.0.1 (x86_64)

full description:

2.7.8 |Anaconda 2.0.1 (x86_64)| (default, Aug 21 2014, 15:21:46) \n[GCC 4.2.1 (Apple Inc. build 5577)]

'''

np.random.seed(12345)

class GenerateDate():

	def __init__(self, nber_servers, 

		maintainanceCost, 

		maintainance_hours,

		software_init_cost, 

		software_update_cost,

		network_hardware_cost,

		installation_cost,

		one_time_cost,

		platform_cost_hour,

		monthly_cost_instance,

		cost_extra_gb_memory,

		cost_extra_gb_storage,

		number_years,

		number_cores,

		memory_size,

		storage_size,

		storage_type,

		usage_hours):

		self.number_servers = nber_servers

		self.basedirs = ["Cat1", "Cat2"]

		self.dfservers = pd.DataFrame()

		self.maintainanceCost = maintainanceCost

		self.maintainance_hours = maintainance_hours

		self.software_init_cost = software_init_cost

		self.software_update_cost = software_update_cost

		self.network_hardware_cost= network_hardware_cost

		self.installation_cost = installation_cost

		self.one_time_cost = one_time_cost

		self.platform_cost_hour = platform_cost_hour

		self.monthly_cost_instance = monthly_cost_instance

		self.cost_extra_gb_memory = cost_extra_gb_memory #cost per extra giga bite of memory

		self.cost_extra_gb_storage = cost_extra_gb_storage

		self.number_years = number_years

		self.number_cores = number_cores

		self.memory_size = memory_size

		self.storage_size = storage_size

		self.storage_type = storage_type

		self.usage_hours = usage_hours

	def duplicate(self, x, n):

		return [x[0]] *n

	def getItem(self, item):

		if item == "procs":

			return "procUnitCost"

		else:

			if item == "mem":

				return "MemUnitCost"

			else:

				if item == "storage3inch" or item == "storage2inch":

					return "StorageUnitCost"

				else:

					if item == "network":

						return "netUnitCost"

	def difference_percentage(self, x):

		if np.nan in x:

			return np.nan

		else:

			return round((x[0] / x[1]) * 100, 2)

	'''

		base configuration data frame

		file name for the corresponding extension file

		flag to identify the appropriate component

	'''

	def createServerSet(self, baseConfDf, extensionfile):

		xcl_file = pd.ExcelFile(extensionfile)

		dframes = {sheet_name: xcl_file.parse(sheet_name) for sheet_name in xcl_file.sheet_names}

		#

		keys = dframes.keys()

		## add the processor unit cost

		baseConfDf[self.getItem("procs")] = 0

		#

		data = baseConfDf.merge(dframes["procs"], how="outer", copy=False)

		colnames = list(data.columns)

		proc_names = list(dframes["procs"])

		colnames_t = [x for x in colnames if x not in proc_names]

		colnames = [x for x in colnames_t if x not in ["id"]]

		data[colnames] = data[colnames].apply(self.duplicate, 0, args=[data.shape[0]])		

		ext = [v for v in keys if v in ["mem", "network","storage3inch", "storage2inch"]]

		print ext

		if ("storage3inch" in ext) and ("storage2inch" in ext):

			ext.remove("storage3inch")