Need Help(live update)

Dash Python
1

Hi Experts,
I have a requirement to create a dashboard which should show live dashboard and also have a feature which updates same graph if we give start and end dates and click submit(pulls the data between the range and updates the graph).

dashboard layout

start date : date picker end date : date picker submit button

graph1

graph2

Requirement :

  1. update live dashboard for every x minutes
  2. if start date and end date is given and submit button is clicked fetch data and update dashboard.

problem i am facing:
i am able to achieve live dash board if i dont pass the button click input, but when tried to pass button click in below callback method, it is not updating the live graph, only second requirement is fulfilled.

import dash as dash
import dash_core_components as dcc
import dash_html_components as html
import datetime as datetime
external_stylesheets = [‘https://codepen.io/chriddyp/pen/bWLwgP.css’]
app = dash.Dash(name, external_stylesheets=external_stylesheets)
from dash.dependencies import Input, Output
global fig3
def serve_layout():
import pandas as pd
str_url = ‘http://x.x.x.x.3:50070/explorer.html#/apps/hive/warehouse/testhivedrivertable/kpidate=’
date_input= str(datetime.date.today())
hdfs_url = str_url + date_input
current_time = datetime.datetime.now() # use datetime.datetime.utcnow() for UTC time
current_time = current_time - datetime.timedelta(seconds=current_time.second)
ten_minutes_ago = current_time - datetime.timedelta(minutes=600)
end_time = int(current_time.timestamp() * 1000)
start_time = int(ten_minutes_ago.timestamp() * 1000)
df = data_collector(hdfs_url,start_time,end_time,[“timestamp”,“hostname”,“os_param”,“value”,“Response Time”,“Count”])
df1 = df.pivot_table(index=[‘timestamp’],values=[‘Count’],aggfunc={‘Count’: np.sum}).fillna(‘0’).reset_index()
df2 = df.pivot_table(index=[‘timestamp’,‘hostname’],values=[‘Count’],aggfunc={‘Count’: np.sum}).fillna(‘0’).reset_index()
# In[7]:

# In[9]:


res = sm.tsa.seasonal_decompose(np.asarray(df1['Count']), freq=5)
Res = res.resid
Value_Cnt = np.count_nonzero(np.nan_to_num(Res))


# In[10]:



# In[11]:


resnew = pd.DataFrame(res.resid)


# In[12]:

resnew2 = pd.DataFrame(res.resid, columns = ['value'])

# In[13]:

resnew2 = resnew.mean(skipna=True)

# In[14]:


resnew3 =  resnew.std(skipna=True)


# In[15]:


thres = resnew2  - 4 * resnew3
uthres = resnew2  + 4* resnew3


# In[16]:


test = uthres[0]
test2 = thres[0]


# In[17]:


df_new = pd.DataFrame(res.resid,columns=['value']).fillna('0').reset_index()


# In[18]:


df_new['anomaly'] = 0


# In[19]:


df_new.loc[(pd.to_numeric(df_new.value) > pd.to_numeric(test)), 'anomaly'] = 1 


# In[20]:


df_new.loc[(pd.to_numeric(df_new.value) < pd.to_numeric(test2)), 'anomaly'] = 1


# In[21]:


df_new2 =  pd.concat((df1,df_new['anomaly']),axis = 1 )


# In[22]:


ls2 = df2.hostname.unique()
df_norm = pd.DataFrame()
df_norm2 =  pd.DataFrame()
df_new_sub = pd.DataFrame(res.resid,columns=['value']).fillna('0').reset_index()
my_df2= []
df_new_sub['anomaly'] = 0


# In[23]:


for i in ls2:
    subres1 = sm.tsa.seasonal_decompose(np.asarray(df2.loc[df2['hostname'] == i].Count), freq=5)
    subRes2 = res.resid
    subValue_Cnt = np.count_nonzero(np.nan_to_num(subRes2))
    subresnew = pd.DataFrame(res.resid)
    subresnew2 = pd.DataFrame(res.resid, columns = ['value'])
    subresnew2 = subresnew.mean(skipna=True)
    subresnew3 =  subresnew.std(skipna=True)
    subthres = subresnew2  - 4 * subresnew3
    subuthres = subresnew2  + 4 * subresnew3
    subtest = subuthres[0]
    subtest2 = subthres[0]
    df_new_sub.loc[(pd.to_numeric(df_new_sub.value) > pd.to_numeric(subtest)), 'anomaly'] = 1 
    df_new_sub.loc[(pd.to_numeric(df_new_sub.value) < pd.to_numeric(subtest2)), 'anomaly'] = 1 
    df_norm = df_norm.append(df_new_sub)    


# In[24]:


df_norm.reset_index(drop=True, inplace=True)
df_new3 =  pd.concat((df_norm['anomaly'],df2),axis = 1 )

ls4 = df_new3.loc[df_new3['anomaly'] == 1].hostname.unique()
ls5 = df_new3.hostname.unique()


anomly_cnt = df_new.loc[df_new['anomaly'] == 1].anomaly.count()


import plotly.plotly as py
import plotly.graph_objs as go
data2 = []

import pandas as pd
##
fig3 = tools.make_subplots(rows=ls2.size+2, cols=1)
##
if(len(ls4) > 0):
    k = 1
    for i in range(len(ls4)):
        j = ls4[i] 
        j = Scatter(x=df2.loc[df2['hostname'] == ls4[i]].timestamp,
                    y=df2.loc[df2['hostname'] == ls4[i]].Count,
                    mode = 'lines',
                    name = str(ls4[i]) ,
                    opacity = 0.9)
        fig3.append_trace(j, i+k, 1)
        data2.append(j) 
        k = 0
if(len(ls4) == 0):
    k = 1
    for i in range(len(ls5)):
        j = ls5[i] 
        j = Scatter(x=df2.loc[df2['hostname'] == ls5[i]].timestamp,
                    y=df2.loc[df2['hostname'] == ls5[i]].Count,
                    mode = 'lines',
                    name = str(ls5[i]) ,
                    opacity = 0.9)
        fig3.append_trace(j, i+k, 1)
        data2.append(j) 
        k = 0
layout = dict(
   
    title = 'Average number of requests per minute', height=500,width=1320,titlefont=dict(
            family='Times New Roman',size=15,color='#7f7f7f'),

    xaxis = dict(rangeselector=dict(title ='Time',
                    ),
        rangeslider=dict(title ='Time',
            visible = True
        ),
        type='date'),
    yaxis={'title': 'Requests Count'},
           
)

fig3 = dict(data=data2, layout=layout)

figure={
            'data': [
                {'x': df_new2.timestamp, 'y': df_new2['Count'], 'type': 'Scatter', 'name': 'Request Cnt:'+ str(df_new2['Count'].sum()),'color' : '#17BECF'},
                {'x': df_new2.loc[df_new2['anomaly'] == 1].timestamp, 'y': df_new2.loc[df_new2['anomaly'] == 1].Count, 'type': 'Scatter','mode':'markers', 'name': 'Outliers Cnt:#'+ str(df_new2[df_new2['anomaly'] == 1].count()["Count"]),'marker': dict(color='red', size=30,symbol ='circle-open-dot')},
            ],
            'layout': go.Layout(
                title = 'Average number of requests per minute',titlefont=dict(family='Times New Roman',size=15,color='#7f7f7f'),
                height=500,
                width=1320,
                xaxis = dict(title ='Time',rangeslider=dict(
                visible = True
            ),
            type='date'),
                yaxis={'title': 'Requests Count'},
                #plot_bgcolor="#000000",

                hovermode='closest')
        }


# In[ ]:
print("hi")
return  html.Div([     
    html.Div([
        html.Div([
            html.H3('Dashboard',style = {'font-size': '27px',
    'background-color': '#005AAC',
    'height': '41px',
    'color': '#FFFFFF',
    'padding-top': '8px',
    'padding-left': '10px',})
        ], className = "twelve.columns columns"),

    ], className = "row"),

         html.Div([
         html.Div([ 
            html.Div([
            html.Label(('Start Time:'),style={ 'display': 'inline','float': 'left','line-height': '2.3em'}),
            dcc.Input(
            id='start-time-input',
            type='datetime-local',
            value=datetime.datetime.strftime(datetime.datetime.utcnow() - pd.offsets.Minute(10), '%Y-%m-%dT%H:%M'),
            className='form-control',style = { 'float': 'left',
            'margin-right': '10px',
            'margin-bottom': '10px',
            'height': '31px','border-radius':'5px','border-style':'ridge'},
            ),
            ],id='div-start-time',style={ 'width': '25%','float': 'left'}),
            html.Div([
            html.Label(('End Time:'),style={ 'left': 'inline','display': 'inline','float': 'left'}),
            dcc.Input(
            id='end-time-input',
            type='datetime-local',
            value=datetime.datetime.strftime(datetime.datetime.utcnow() - pd.offsets.Minute(10), '%Y-%m-%dT%H:%M'),
            className='form-control',style = {'float': 'left',
            'margin-bottom': '3px','height': '31px','border-radius':'5px','border-style':'ridge'},
            ),
            ],style={  'width':'24%' , 'float': 'left'}),
            html.Label(('Sensitivity:'),style={ 'left': 'inline','display': 'inline','float': 'left'}),
            dcc.Dropdown(
            id='sensitivity',
            options=[
                {'label': 'High', 'value': '4'},
                {'label': 'Medium', 'value': '3'},
                {'label': 'Low', 'value': '2'}
            ],
            value='4', style = { 'float': 'left','height' :'10px !important','width':'90px','margin-right':'5px','border-radius':'5px'},),
            html.Button('Submit', id='button', style = {'padding': '5px 5px 5px 5px','line-height': '0px','border-style':'ridge','height': '34px','width': '100px'}),
            html.Div(id='hidden-div', style={'display':'none'}),

         ]),

        html.Div([
             dcc.Interval(id='interval-component',interval=1*1000,n_intervals=0),
             dcc.Graph(
                id = 'graph-1',
                #animate=True,
                figure = figure,
                config = {
                    'doubleClick': 'reset'
                }
            )
        ], className = 'twelve columns')
    ], className ='row',style={

        'backgroundColor': 'rgb(234, 233, 232)',
        'padding': '10px 5px'}),

    html.Div([
        html.Div([

            dcc.Graph(
                id = 'graph-2',
                #animate=True,
                figure = fig3
            )
        ], className = 'twelve columns')
    ], className ='row',style={

        'backgroundColor': 'rgb(234, 233, 232)',
        'padding': '10px 5px'}),
])

app.layout = serve_layout
#First CallBack - call to update first graph -returns graph1
@app.callback(Output(component_id=‘graph-1’, component_property=‘figure’),
[Input(‘button’, ‘n_clicks’),
Input(component_id=‘start-time-input’, component_property=‘value’),
Input(component_id= ‘end-time-input’, component_property=‘value’),
Input(component_id= ‘sensitivity’, component_property=‘value’)] )
def update_graph_live(n_clicks,stime,etime,sens):
import pandas as pd
if pd.to_numeric(n_clicks) > 0:
import pandas as pd
str_url = ‘http://x.x.x.x:xxxx/explorer.html#/apps/hive/warehouse/testhivedrivertable/kpidate=’

date_input= str(datetime.date.today())

        hdfs_url = str_url +  stime.split("T")[0]
        current_time = datetime.datetime.now()  # use datetime.datetime.utcnow() for UTC time
        current_time = current_time - datetime.timedelta(seconds=current_time.second)
        ten_minutes_ago = current_time - datetime.timedelta(minutes=600)
        end_time =   int(datetime.datetime.strptime(etime, '%Y-%m-%dT%H:%M').timestamp())*1000
        start_time = int(datetime.datetime.strptime(stime, '%Y-%m-%dT%H:%M').timestamp())*1000
        
        df = data_collector(hdfs_url,start_time,end_time,["timestamp","hostname","os_param","value","Response Time","Count"])
        df1 = df.pivot_table(index=['timestamp'],values=['Count'],aggfunc={'Count': np.sum}).fillna('0').reset_index()
        df2 = df.pivot_table(index=['timestamp','hostname'],values=['Count'],aggfunc={'Count': np.sum}).fillna('0').reset_index()
        # In[7]:

        # In[9]:


        res = sm.tsa.seasonal_decompose(np.asarray(df1['Count']), freq=5)
        Res = res.resid
        Value_Cnt = np.count_nonzero(np.nan_to_num(Res))


        # In[10]:



        # In[11]:


        resnew = pd.DataFrame(res.resid)


        # In[12]:


        resnew2 = pd.DataFrame(res.resid, columns = ['value'])


        # In[13]:


        resnew2 = resnew.mean(skipna=True)


        # In[14]:


        resnew3 =  resnew.std(skipna=True)


        # In[15]:


        thres = resnew2  - pd.to_numeric(sens) * resnew3
        uthres = resnew2  + pd.to_numeric(sens)* resnew3


        # In[16]:


        test = uthres[0]
        test2 = thres[0]


        # In[17]:


        df_new = pd.DataFrame(res.resid,columns=['value']).fillna('0').reset_index()


        # In[18]:


        df_new['anomaly'] = 0


        # In[19]:


        df_new.loc[(pd.to_numeric(df_new.value) > pd.to_numeric(test)), 'anomaly'] = 1 


        # In[20]:


        df_new.loc[(pd.to_numeric(df_new.value) < pd.to_numeric(test2)), 'anomaly'] = 1


        # In[21]:


        df_new2 =  pd.concat((df1,df_new['anomaly']),axis = 1 )


        # In[22]:


        ls2 = df2.hostname.unique()
        df_norm = pd.DataFrame()
        df_norm2 =  pd.DataFrame()
        df_new_sub = pd.DataFrame(res.resid,columns=['value']).fillna('0').reset_index()
        my_df2= []
        df_new_sub['anomaly'] = 0


        # In[23]:

        import plotly.plotly as py
        import plotly.graph_objs as go
        data2 = []

        import pandas as pd
        ##
        figure={
                    'data': [
                        {'x': df_new2.timestamp, 'y': df_new2['Count'], 'type': 'Scatter', 'name': 'Request Cnt:'+ str(df_new2['Count'].sum()),'color' : '#17BECF'},
                        {'x': df_new2.loc[df_new2['anomaly'] == 1].timestamp, 'y': df_new2.loc[df_new2['anomaly'] == 1].Count, 'type': 'Scatter','mode':'markers', 'name': 'Outliers Cnt:#'+ str(df_new2[df_new2['anomaly'] == 1].count()["Count"]),'marker': dict(color='red', size=30,symbol ='circle-open-dot')},
                    ],
                    'layout': go.Layout(
                        title = 'Average number of requests per minute',titlefont=dict(family='Times New Roman',size=15,color='#7f7f7f'),
                        height=500,
                        width=1320,
                        xaxis = dict(title ='Time',rangeselector=dict(
                                ),
                    rangeslider=dict(
                        visible = True
                    ),
                    type='date'),
                        yaxis={'title': 'Requests Count'},
                        #plot_bgcolor="#000000",

                        hovermode='closest')
                }

        return figure

#Second CallBack - call to update Second graph -returns graph2
@app.callback(Output(component_id=‘graph-2’, component_property=‘figure’),
[Input(‘button’, ‘n_clicks’),
Input(component_id=‘start-time-input’, component_property=‘value’),
Input(component_id= ‘end-time-input’, component_property=‘value’),
Input(component_id= ‘sensitivity’, component_property=‘value’)] )
def update_graph_live2(n_clicks,stime,etime,sens):
import pandas as pd
if pd.to_numeric(n_clicks) > 0:
import pandas as pd
str_url = ‘http://x.x.x.x.x:xxxx/explorer.html#/apps/hive/warehouse/testhivedrivertable/kpidate=’

date_input= stime.split(“T”)[0]

        hdfs_url = str_url + stime.split("T")[0]
        end_time =   int(datetime.datetime.strptime(etime, '%Y-%m-%dT%H:%M').timestamp())*1000
        start_time = int(datetime.datetime.strptime(stime, '%Y-%m-%dT%H:%M').timestamp())*1000
        df = data_collector(hdfs_url,start_time,end_time,["timestamp","hostname","os_param","value","Response Time","Count"])
        df1 = df.pivot_table(index=['timestamp'],values=['Count'],aggfunc={'Count': np.sum}).fillna('0').reset_index()
        df2 = df.pivot_table(index=['timestamp','hostname'],values=['Count'],aggfunc={'Count': np.sum}).fillna('0').reset_index()
        # In[7]:

        # In[9]:
        res = sm.tsa.seasonal_decompose(np.asarray(df1['Count']), freq=5)
        Res = res.resid

        # In[10]:



        # In[11]:


        resnew = pd.DataFrame(res.resid)


        # In[12]:


        resnew2 = pd.DataFrame(res.resid, columns = ['value'])


        # In[13]:


        resnew2 = resnew.mean(skipna=True)


        # In[14]:


        resnew3 =  resnew.std(skipna=True)


        # In[15]:


        thres = resnew2  - pd.to_numeric(sens) * resnew3
        uthres = resnew2  + pd.to_numeric(sens)* resnew3


        # In[16]:


        test = uthres[0]
        test2 = thres[0]


        # In[17]:


        df_new = pd.DataFrame(res.resid,columns=['value']).fillna('0').reset_index()


        # In[18]:


        df_new['anomaly'] = 0


        # In[19]:


        df_new.loc[(pd.to_numeric(df_new.value) > pd.to_numeric(test)), 'anomaly'] = 1 


        # In[20]:


        df_new.loc[(pd.to_numeric(df_new.value) < pd.to_numeric(test2)), 'anomaly'] = 1


        # In[21]:


        df_new2 =  pd.concat((df1,df_new['anomaly']),axis = 1 )


        # In[22]:


        ls2 = df2.hostname.unique()
        df_norm = pd.DataFrame()
        df_norm2 =  pd.DataFrame()
        df_new_sub = pd.DataFrame(res.resid,columns=['value']).fillna('0').reset_index()
        my_df2= []
        df_new_sub['anomaly'] = 0


        # In[23]:


        for i in ls2:
            subres1 = sm.tsa.seasonal_decompose(np.asarray(df2.loc[df2['hostname'] == i].Count), freq=5)
            subRes2 = res.resid
            subValue_Cnt = np.count_nonzero(np.nan_to_num(subRes2))
            subresnew = pd.DataFrame(res.resid)
            subresnew2 = pd.DataFrame(res.resid, columns = ['value'])
            subresnew2 = subresnew.mean(skipna=True)
            subresnew3 =  subresnew.std(skipna=True)
            subthres = subresnew2  - pd.to_numeric(sens) * subresnew3
            subuthres = subresnew2  + pd.to_numeric(sens) * subresnew3
            subtest = subuthres[0]
            subtest2 = subthres[0]
            df_new_sub.loc[(pd.to_numeric(df_new_sub.value) > pd.to_numeric(subtest)), 'anomaly'] = 1 
            df_new_sub.loc[(pd.to_numeric(df_new_sub.value) < pd.to_numeric(subtest2)), 'anomaly'] = 1 
            df_norm = df_norm.append(df_new_sub)    


        # In[24]:
        df_norm.reset_index(drop=True, inplace=True)
        df_new3 =  pd.concat((df_norm['anomaly'],df2),axis = 1 )

        ls4 = df_new3.loc[df_new3['anomaly'] == 1].hostname.unique()
        ls5 = df_new3.hostname.unique()

        import plotly.plotly as py
        import plotly.graph_objs as go
        data2 = []

        import pandas as pd
        ##
        fig3 = tools.make_subplots(rows=ls2.size+2, cols=1)
        ##
        if(len(ls4) > 0):
            k = 1
            for i in range(len(ls4)):
                j = ls4[i] 
                j = Scatter(x=df2.loc[df2['hostname'] == ls4[i]].timestamp,
                            y=df2.loc[df2['hostname'] == ls4[i]].Count,
                            mode = 'lines',
                            name = str(ls4[i]) ,
                            opacity = 0.9)
                fig3.append_trace(j, i+k, 1)
                data2.append(j) 
                k = 0
        if(len(ls4) == 0):
            k = 1
            for i in range(len(ls5)):
                j = ls5[i] 
                j = Scatter(x=df2.loc[df2['hostname'] == ls5[i]].timestamp,
                            y=df2.loc[df2['hostname'] == ls5[i]].Count,
                            mode = 'lines',
                            name = str(ls5[i]) ,
                            opacity = 0.9)
                fig3.append_trace(j, i+k, 1)
                data2.append(j) 
                k = 0
        layout = dict(

            title = 'Average number of requests per minute', height=500,width=1320,titlefont=dict(
                    family='Times New Roman',size=15,color='#7f7f7f'),

            xaxis = dict(rangeslider=dict(title ='Time',
                    visible = True
                ),
                type='date'),
            yaxis={'title': 'Requests Count'},

        )

        fig3 = dict(data=data2, layout=layout)

        return fig3

if name == ‘main’:
app.run_server(host = ‘x.x.x.x.x’, port = 33071, threaded = True)