Commit f07f2975 authored by Robert Ross's avatar Robert Ross

Merge branch '116-july-open-data-update' into 'master'

Resolve "July Open Data Update"

Closes #116

See merge request !65
parents 9cc17efd 41fbe15d
This diff is collapsed.
# This script loops over each township in current assesment triennial and create first-pass value open data.
library(plyr)
library(dplyr)
#Define the townships in current assesment triennial
curr_towns <- c(26, 17, 23, 16, 22, 25, 10, 20, 38, 29, 35, 24, 18)
scopes <- c()
for (num in curr_towns){
scope_file <- paste0("scope_", num, "_2019.Rda")
scopes <- append(scopes, scope_file, after=length(scopes))
}
# Define the column names which we need and their appropriate names for opendata
full_cols <- c("PIN","ADDR", "most_recent_sale_date",
"DOC_NO", "most_recent_sale_price",
"puremarket", "PRI_EST_LAND",
"PRI_EST_BLDG", "fitted_value_1",
"fitted_value_2", "land_value",
"improvement_value", "fitted_value_3",
"fitted_value_4", "fitted_value_5",
"fitted_value_6", "fitted_value_7")
target_cols <- c("PIN", "Property Address","Most Recent Date Sale",
"Deed No.", "Most Recent Sale Price","Pure Market Indicator",
"Estimate (Land)", "Estimate (Building)", "First Pass Value 1",
"First Pass Value 2", "Land Value", "Improvement Value",
"First Pass Value 3", "First Pass Value 4", "First Pass Value 5",
"First Pass Value 6", "First Pass Value 7")
# Update the opendata
agg_df <- NULL
file_names <- list.files("O:/CCAODATA/results/final_values/")
for (scope in scopes){
curnum <- substring(scope, 7, 8)
if (!(scope %in% file_names)){
print(paste("Scope file does not exist for township code", curnum))}
else {
curfile <- paste("O:/CCAODATA/results/final_values/", scope, sep="")
e <- new.env()
load(curfile, e)
if (!is.null(e$"file_info")){
print(curnum)
file_info <- print_file_info(curnum)
actions <- c()
for (action in file_info["action"]){
actions <- append(actions, as.character(action), after=length(actions))
}
if (!("8_complete_firstpass" %in% actions)){
print(paste("Township code", curnum, "is not complete for first-pass value"))
}else{
valuation_df <- e$valuationdata
valuation_df <- valuation_df %>%
dplyr::select(one_of(full_cols))
i1 <- match(colnames(valuation_df), full_cols, nomatch = 0)
colnames(valuation_df)[i1] <- target_cols[i1]
if (is.null(agg_df)){
agg_df <- valuation_df
}else{
agg_df <- agg_df %>%
rbind.fill(valuation_df)
}}
}
}
}
# Save first-pass values in csv file
write_csv(agg_df, "O:/CCAODATA/output_data/open_data/first_pass_values.csv")
\ No newline at end of file
## This file is a script to revise data dictionary for model data and assessment data.
## Load current data dictionary
model_dic <- read_csv(paste0("C:/Users/",user,"/Documents/ccao_sf_cama_dev/data_dictionary_constituents/modeldata.csv"))
ass_dic <- read_csv(paste0("C:/Users/",user,"/Documents/ccao_sf_cama_dev/data_dictionary_constituents/assmntdata.csv"))
## Revise modeldata dictionary
# Build data dictionary for modeldata
Source.File <- "0_build_200class_modeldata.R"
Column.Name <- colnames(modeldata)
model_exs <- c()
for (i in colnames(modeldata))
{
for (j in nrow(modeldata))
{
if (is.na(modeldata[j, i]) == FALSE)
{
model_exs <- append(model_exs, modeldata[j, i], after=length(model_exs))
break
}
model_exs <- append(model_exs, NA, after=length(model_exs))
}
}
Column.Example <- unlist(model_exs, use.names=FALSE)
Display <- NA
Display.Name <- NA
Description <- NA
types<-c()
for (i in model_exs){
if (class(i) == "numeric"){
types <- append(types, "Number", after=length(types))
}else if(class(i) == "Date"){
types <- append(types, "Date", after=length(types))
}else{
types <- append(types, "Text", after=length(types))
}
}
Data.Type <- types
# Add new rows to current modeldata dictionary
revdic_full <- data.frame(Source.File, Column.Name, Column.Example, Display, Display.Name, Description, Data.Type)
revdic_onlynew <- revdic_full %>%
anti_join(model_dic, by = "Column.Name")
newmodel_dic <- model_dic %>%
rbind(revdic_onlynew)
## Revise assessment dictionary
# Build data dictionary for assmntdata
Source.File <- "0_build_200class_assmntdata.R"
Column.Name <- colnames(assmntdata)
ass_exs <- c()
for (i in colnames(assmntdata))
{
for (j in nrow(assmntdata))
{
if (is.na(assmntdata[j, i]) == FALSE)
{
ass_exs <- append(ass_exs, assmntdata[j, i], after=length(ass_exs))
break
}
ass_exs <- append(ass_exs, NA, after=length(ass_exs))
}
}
Column.Example <- unlist(ass_exs, use.names=FALSE)
Display <- NA
Display.Name <- NA
Description <- NA
types<-c()
for (i in ass_exs){
if (class(i) == "numeric"){
types <- append(types, "Number", after=length(types))
}else if(class(i) == "Date"){
types <- append(types, "Date", after=length(types))
}else{
types <- append(types, "Text", after=length(types))
}
}
Data.Type <- types
# Add new rows to current assmntdata dictionary
revassdic_full <- data.frame(Source.File, Column.Name, Column.Example, Display, Display.Name, Description, Data.Type)
revassdic_onlynew <- revassdic_full %>%
anti_join(ass_dic, by = "Column.Name")
newass_dic <- ass_dic %>%
rbind(revassdic_onlynew)
## Output new dictionaries
write_csv(newmodel_dic, paste0("C:/Users/",user,"/Documents/ccao_sf_cama_dev/data_dictionary_constituents/modeldata_rev.csv"))
write_csv(newass_dic, paste0("C:/Users/",user,"/Documents/ccao_sf_cama_dev/data_dictionary_constituents/assmntdata_rev.csv"))
This diff is collapsed.
## This file is a script to revise data dictionary for model data and assessment data.
library(tidyverse)
library(dplyr)
library(tidyr)
## Load newly created data and current data dictionary
modeldata <- read_csv("C:\\Users\\tando\\Documents\\ccao_sf_cama_dev\\modeldata_20190701.csv")
assmntdata <- read_csv("C:\\Users\\tando\\Documents\\ccao_sf_cama_dev\\assmntdata_20190701.csv")
model_dic <- read_csv("C:\\Users\\tando\\Documents\\ccao_sf_cama_dev\\data_dictionary_constituents\\modeldata.csv")
ass_dic <- read_csv("C:\\Users\\tando\\Documents\\ccao_sf_cama_dev\\data_dictionary_constituents\\assmntdata.csv")
## Revise modeldata dictionary
# Build data dictionary for modeldata
Source.File <- "0_build_200class_modeldata.R"
Column.Name <- colnames(modeldata)
model_exs <- c()
for (i in colnames(modeldata))
{
for (j in nrow(modeldata))
{
if (is.na(modeldata[j, i]) == FALSE)
{
model_exs <- append(model_exs, modeldata[j, i], after=length(model_exs))
break
}
model_exs <- append(model_exs, NA, after=length(model_exs))
}
}
Column.Example <- unlist(model_exs, use.names=FALSE)
Display <- NA
Display.Name <- NA
Description <- NA
types<-c()
for (i in model_exs){
if (class(i) == "numeric"){
types <- append(types, "Number", after=length(types))
}else if(class(i) == "Date"){
types <- append(types, "Date", after=length(types))
}else{
types <- append(types, "Text", after=length(types))
}
}
Data.Type <- types
# Add new rows to current modeldata dictionary
revdic_full <- data.frame(Source.File, Column.Name, Column.Example, Display, Display.Name, Description, Data.Type)
revdic_onlynew <- revdic_full %>%
anti_join(model_dic, by = "Column.Name")
newmodel_dic <- model_dic %>%
rbind(revdic_onlynew)
## Revise assessment dictionary
# Build data dictionary for assmntdata
Source.File <- "0_build_200class_assmntdata.R"
Column.Name <- colnames(assmntdata)
ass_exs <- c()
for (i in colnames(assmntdata))
{
for (j in nrow(assmntdata))
{
if (is.na(assmntdata[j, i]) == FALSE)
{
ass_exs <- append(ass_exs, assmntdata[j, i], after=length(ass_exs))
break
}
ass_exs <- append(ass_exs, NA, after=length(ass_exs))
}
}
Column.Example <- unlist(ass_exs, use.names=FALSE)
Display <- NA
Display.Name <- NA
Description <- NA
types<-c()
for (i in ass_exs){
if (class(i) == "numeric"){
types <- append(types, "Number", after=length(types))
}else if(class(i) == "Date"){
types <- append(types, "Date", after=length(types))
}else{
types <- append(types, "Text", after=length(types))
}
}
Data.Type <- types
# Add new rows to current assmntdata dictionary
revassdic_full <- data.frame(Source.File, Column.Name, Column.Example, Display, Display.Name, Description, Data.Type)
revassdic_onlynew <- revassdic_full %>%
anti_join(ass_dic, by = "Column.Name")
newass_dic <- ass_dic %>%
rbind(revassdic_onlynew)
## Output new dictionaries
write_csv(newmodel_dic, "C:\\Users\\tando\\Documents\\ccao_sf_cama_dev\\data_dictionary_constituents\\modeldata.csv")
write_csv(newass_dic, "C:\\Users\\tando\\Documents\\ccao_sf_cama_dev\\data_dictionary_constituents\\assmntdata.csv")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment