ServerEvents stop working after more than 12 hours

Hi guys,
I have a strange issue. This is my AppHost

public class MaresAppHost : AppSelfHostBase
    {
        public MaresAppHost(Assembly[] assembliesWithServices)
            : base("Mares API", assembliesWithServices)
        {
            MaresEnvironment.Instance.RegisterLicense();
        }

        public string HostUrl => $"http://*:{ApplicationVariables.Instance.APIPort}/";

        public override void Configure(Container container)
        {
            ServiceEnvironment.Container = container;

            Plugins.Add(new ServerEventsFeature());
            //Plugins.Add(new SwaggerFeature());

            //registro tutte le rotte
            Routes.AddFromAssembly(base.ServiceAssemblies.ToArray());

            var redisHost = ApplicationVariables.Instance.RedisHost;
            if (!string.IsNullOrEmpty(redisHost))
            {
                container.Register<IRedisClientsManager>(
                    new RedisManagerPool(redisHost));

                container.Register<IServerEvents>(c =>
                    new RedisServerEvents(c.Resolve<IRedisClientsManager>()));

                container.Resolve<IServerEvents>().Start();
            }


            //Configs the host
            SetConfig(new HostConfig
            {
                DebugMode = true,
                EnableFeatures = Feature.All,
            });

            //Prevents Json serialized to add __type part on json string for anonymous types serialization
            //Read this: -http://stackoverflow.com/questions/18842685/servicestack-sessions-doesnt-work-when-using-jsconfig-excludetypeinfo
            JsConfig.ExcludeTypeInfo = true;
            JsConfig.DateHandler = DateHandler.ISO8601;
        }
    }

and this is my “wrapper” around the ServiceEventClient

public class MaresApiProxy : IDisposable
    {
        #region Constructors

        public MaresApiProxy():this("http://localhost:50000")
        { }

        public MaresApiProxy(string url)
        {
            this.Url = url;
            this.EventClient = new ServerEventsClient(url);
        }

        #endregion

        #region Fields & Properties

        public string Url { get; private set; }
        public ServerEventsClient EventClient { get; private set; }

        protected static string SsidCookie { get; set; }

        public string Status => this.EventClient?.Status;
        public bool? IsStopped => this.EventClient?.IsStopped;
        public DateTime? LastPulseAt => this.EventClient?.LastPulseAt;


        #endregion

        #region Methods

        internal async Task<T> PostNotification<T>(T obj)
            where T : INotificationBase, new()
        {
            JsonServiceClient client = null;
            try
            {

                client = new JsonServiceClient(this.Url);
                return await client.PostAsync<T>(obj);
            }
            catch (WebServiceException ex)
            {
                return this.GetServiceExceptionResponseStatus<T>(ex);
            }
            catch (Exception ex)
            {
                return this.GetExceptionResponseStatus<T>(ex);
            }
            finally
            {
                if (client != null)
                {
                    client.Dispose();
                    client = null;
                }
            }
        }

        internal T Post<T>(T obj)
            where T : IEntityResult, new()
        {
            JsonServiceClient client = null;
            try
            {

                client = new JsonServiceClient(this.Url);
                return client.Post<T>(obj);
            }
            catch (WebServiceException ex)
            {
                return this.GetServiceExceptionResponseStatus<T>(ex);
            }
            catch (Exception ex)
            {
                return this.GetExceptionResponseStatus<T>(ex);
            }
            finally
            {
                if (client != null)
                {
                    client.Dispose();
                    client = null;
                }
            }
        }

        internal T Get<T, K>(K request)
            where T : IEntityResult, new()
        {
            JsonServiceClient client = null;
            try
            {
                client = new JsonServiceClient(Url);
                T response = client.Get<T>(request);
                return response;
            }
            catch (WebServiceException ex)
            {
                return this.GetServiceExceptionResponseStatus<T>(ex);
            }
            catch (Exception ex)
            {
                return this.GetExceptionResponseStatus<T>(ex);
            }
            finally
            {
                if (client != null)
                {
                    client.Dispose();
                    client = null;
                }
            }
        }

        internal async Task<T> GetAsync<T, K>(K request)
            where T : IEntityResult, new()
        {
            JsonServiceClient client = null;
            try
            {
                client = new JsonServiceClient(Url);
                T response = await client.GetAsync<T>(request);
                return response;
            }
            catch (WebServiceException ex)
            {
                return this.GetServiceExceptionResponseStatus<T>(ex);
            }
            catch (Exception ex)
            {
                return this.GetExceptionResponseStatus<T>(ex);
            }
            finally
            {
                if (client != null)
                {
                    client.Dispose();
                    client = null;
                }
            }
        }

        protected T GetServiceExceptionResponseStatus<T>(WebServiceException ex)
            where T : IEntityResult, new()
        {
            T errorResponse = new T();
            errorResponse.HasError = true;
            errorResponse.StatusCode = ex.StatusCode;

            if (ex.ResponseStatus != null)
                errorResponse.ResponseStatus = ex.ResponseStatus;
            else
                errorResponse.ResponseStatus = new ResponseStatus()
                {
                    ErrorCode = ex.ErrorCode,
                    Message = ex.Message,
                    StackTrace = ex.StackTrace
                };
            return errorResponse;
        }

        protected T GetExceptionResponseStatus<T>(Exception ex)
             where T : IEntityResult, new()
        {
            T errorResponse = new T();
            errorResponse.HasError = true;
            errorResponse.StatusCode = 500;

            errorResponse.ResponseStatus = new ResponseStatus()
            {
                ErrorCode = "500",
                Message = ex.Message,
                StackTrace = ex.StackTrace
            };
            return errorResponse;
        }

        public void Dispose()
        {
            if (this.EventClient == null) return;

            this.EventClient.Dispose();
            this.EventClient = null;
        }

        public void Start()
        {
            this.EventClient.Start();
        }

        public void StartListening(string channel)
        {
            this.EventClient.Update(new[] { channel }, null);
        }

        public void StopListening(string channel)
        {
            this.EventClient.Update(null, new string[] { channel });
        }        

        public void AddListener<T>(Action<T> listenerAction)
        {
            string name = typeof(T).Name;
            this.EventClient.Handlers[name] = (client, msg) =>
            {
                T obj = msg.Json.FromJson<T>();
                listenerAction(obj);
            };
        }

        #endregion
    }

and this is what happen: there are 2 applications, 1 pubblisher and 1 subscriber. The pubblisher push notifications every 500ms and the subscriber display that data on the screen. During the night the subscriber stop to receive notifications, even if the pubblisher still push them, I tryed restarting the subscriber application without luck, restarting pubblisher application without luck, once restarted the apphost everything start to work again. AppHost was active and fully functional, in fact every call to its methods works perfectly, only notifications were not working.

What is the best practice to keep long connection active? I checked the pc, and there aren’t any power off options

Best regards
Enrico

There aren’t any other options, all Server Events Clients should be sending a heartbeat to detect when a connection is no longer active which will trigger it to reconnect.

I’d like to know what the HTTP traffic looked like when it stopped working, whether the notifications were still being sent and whether there were any errors in the heartbeat requests or server event reconnections.

can you please give me some suggention on how to detect this kind of failure?
I got the problem again this afternoon

thanks
Enrico

If you’re saying notifications are no longer working, then to test it not working would be to create a script that sends itself a message to the channel its connected to.

What happens when you try to connect to /event-stream? Do you receive the cmd.onConnect event?

For comparison here’s what a working stream returns: http://chat.netcore.io/event-stream

I’m just wondering if there may be some kind of errors on my implementation. Can you please take a look on my code? This is my Subscriber app code (MaresApiProxy is the class I already gave you)

this.MaresApiProxy = new MaresApiProxy();
this.MaresApiProxy.StartListening(“ABC”);
this.MaresApiProxy.AddListener(x =>
{
//notification stuff here
});
this.MaresApiProxy.Start();

this is my pubblisher app code

            MachineDriverNotification n = new MachineDriverNotification()
            {
                MachineCode = "ABC",
                Data = e.Data
            };
            using (var proxy = new MaresApiProxy())
                proxy.SendNotification(n);

where proxy.SendNotification(n) is

public static async void SendNotification(this MaresApiProxy proxy, MachineDriverNotification request)
    {
        await proxy.PostNotification(request);
    }

where proxy.PostNotification(request) is

    internal async Task<T> PostNotification<T>(T obj)
        where T : INotificationBase, new()
    {
        JsonServiceClient client = null;
        try
        {

            client = new JsonServiceClient(this.Url);
            return await client.PostAsync<T>(obj);
        }
        catch (WebServiceException ex)
        {
            return this.GetServiceExceptionResponseStatus<T>(ex);
        }
        catch (Exception ex)
        {
            return this.GetExceptionResponseStatus<T>(ex);
        }
        finally
        {
            if (client != null)
            {
                client.Dispose();
                client = null;
            }
        }
    }

the apphost code is this

public class StandardService : Service
{
public IServerEvents ServerEvents { get; set; }

    #region REST Methods

    public RequestResult<List<MachineStatistic>> Get(MachineStatisticsGet request)
    {
        var manager = new StandardServiceManager();
        var res = manager.GetMachineStatistics(request);
        return new RequestResult<List<MachineStatistic>>() { Value = res };
    }

    public RequestResult<bool> Get(MachineStatisticUpdate request)
    {
        var manager = new StandardServiceManager();
        var res = manager.UpdateStatisticValue(request);
        return new RequestResult<bool>() { Value = res };
    }

    public RequestResult<MachineStatistic> Get(MachineStatisticGet request)
    {
        var manager = new StandardServiceManager();
        var res = manager.GetStatistic(request);
        return new RequestResult<MachineStatistic>() { Value = res };
    }

    public RequestResult<double> Get(MachineStatisticGetNextValue request)
    {
        var manager = new StandardServiceManager();
        var res = manager.GetNextValue(request);
        return new RequestResult<double>() { Value = res };
    }

    public RequestResult<List<Machine>> Get(MachinesGet request)
    {
        var manager = new StandardServiceManager();
        var res = manager.GetMachines(request);
        return new RequestResult<List<Machine>>() { Value = res };
    }

    public RequestResult<List<MachineStateChanx>> Get(MachineStateChangesGet request)
    {
        var manager = new StandardServiceManager();
        var res = manager.GetMachineStateChanges(request);
        return new RequestResult<List<MachineStateChanx>>() { Value = res };
    }


    #endregion

    #region NOTIFICATION Methods

    private void PostNotification<T>(T request)
        where T : INotificationBase
    {
        if (string.IsNullOrEmpty(request.MachineCode))
            throw new ArgumentNullException("machineCode");

        ServerEvents.NotifyChannel(request.MachineCode, request);
    }

    public void Post(MachineStatisticChangedNotification request)
    {
        this.PostNotification(request);
    }

    public void Post(MachineDriverNotification request)
    {
        this.PostNotification(request);
    }

    public void Post(MachineProgramStatusChangeNotification request)
    {
        this.PostNotification(request);
    }

    public void Post(MachineProgramLineUpdatedNotification request)
    {
        this.PostNotification(request);
    }

    public void Post(MachineToolSpindleChangedNotification request)
    {
        this.PostNotification(request);
    }

    public void Post(MachineStatusChangedNotification request)
    {
        this.PostNotification(request);
    }

    public void Post(MachineMonitoringOpenNotification request)
    {
        this.PostNotification(request);
    }

    public void Post(MaresDriverConnectionInfoNotification request)
    {
        this.PostNotification(request);
    }

    public void Post(MachineSpindleNotification request)
    {
        this.PostNotification(request);
    }

    public void Post(MachineAxisNotification request)
    {
        this.PostNotification(request);
    }

    #endregion
}

Thanks
Enrico

i Monday I will try the /event-stream path and see the result. I just try now, where everything is working and I get this

id: 1
data: cmd.onConnect {“userId”:"-76908",“isAuthenticated”:“false”,“displayName”:“user76908”,“channels”:"*",“createdAt”:“1553276918352”,“profileUrl”:“https://raw.githubusercontent.com/ServiceStack/Assets/master/img/apps/no-profile64.png",“id”:“fQw23fboZhVb93nemeW9”,“unRegisterUrl”:“http://192.168.34.116:50000/event-unregister?id=fQw23fboZhVb93nemeW9”,“heartbeatUrl”:“http://192.168.34.116:50000/event-heartbeat?id=fQw23fboZhVb93nemeW9”,“updateSubscriberUrl”:“http://192.168.34.116:50000/event-subscribers”,“heartbeatIntervalMs”:“10000”,“idleTimeoutMs”:"30000”}

id: 2
data: @cmd.onJoin {“userId”:"-76908",“isAuthenticated”:“false”,“displayName”:“user76908”,“channels”:"",“createdAt”:“1553276918352”,“profileUrl”:“https://raw.githubusercontent.com/ServiceStack/Assets/master/img/apps/no-profile64.png",“channel”:"*”}

I will let you know.
Regards
Enrico

Good afternoon,
Today the problem appear again.

Checking the event-stream path I receive this

id: 1
data: cmd.onConnect {“userId”:"-339",“isAuthenticated”:“false”,“displayName”:“user339”,“channels”:"*",“createdAt”:“1553534393216”,“profileUrl”:“https://raw.githubusercontent.com/ServiceStack/Assets/master/img/apps/no-profile64.png",“id”:“7qNo7EPEfIBPvtJs2zFe”,“unRegisterUrl”:“http://192.168.50.192:50000/event-unregister?id=7qNo7EPEfIBPvtJs2zFe”,“heartbeatUrl”:“http://192.168.50.192:50000/event-heartbeat?id=7qNo7EPEfIBPvtJs2zFe”,“updateSubscriberUrl”:“http://192.168.50.192:50000/event-subscribers”,“heartbeatIntervalMs”:“10000”,“idleTimeoutMs”:"30000”}

id: 2
data: @cmd.onJoin {“userId”:"-339",“isAuthenticated”:“false”,“displayName”:“user339”,“channels”:"",“createdAt”:“1553534393216”,“profileUrl”:“https://raw.githubusercontent.com/ServiceStack/Assets/master/img/apps/no-profile64.png",“channel”:"*”}

I don’t know if it might be of any help, but we use our subscriber application via Remote Desktop, and in the past RDP gave us some issue when sending notification using a TCP/IP socket.

Ciao
Enrico

That suggests /event-stream is still working, you can try specifying your channel, e.g. /event-stream?channels=home and sending notifications to it to see if they’re still being returned in the stream.

Hi, sending

http://192.168.50.189:50000/event-heartbeat?channels=home

i got http error 404. In a working application I receive

id: 1
data: cmd.onConnect {“userId”:"-359",“isAuthenticated”:“false”,“displayName”:“user359”,“channels”:“home”,“createdAt”:“1553535121750”,“profileUrl”:“https://raw.githubusercontent.com/ServiceStack/Assets/master/img/apps/no-profile64.png",“id”:“Hhzmb8ajzkS4GV7yhxDS”,“unRegisterUrl”:“http://192.168.50.192:50000/event-unregister?id=Hhzmb8ajzkS4GV7yhxDS”,“heartbeatUrl”:“http://192.168.50.192:50000/event-heartbeat?id=Hhzmb8ajzkS4GV7yhxDS”,“updateSubscriberUrl”:“http://192.168.50.192:50000/event-subscribers”,“heartbeatIntervalMs”:“10000”,“idleTimeoutMs”:"30000”}

id: 2
data: home@cmd.onJoin {“userId”:"-359",“isAuthenticated”:“false”,“displayName”:“user359”,“channels”:“home”,“createdAt”:“1553535121750”,“profileUrl”:“https://raw.githubusercontent.com/ServiceStack/Assets/master/img/apps/no-profile64.png",“channel”:"home”}

Can you provide the full HTTP Request/Response Headers?

wait… the guys turned off the machine, so the PLC was not reachable and data was not sent because of this.
and for the http error 404 I think I typed the wrong address on the search bar.

Just an info: this moring I found out that I didn’t register the license on the pubblisher application and I think the problem might be this.

Tomorrow I will keep it under investigation all day long. If nothing wrong occur… well… I’m an idiot because of the license

Regards
Enrico

1 Like

Hi guys, it is proved now! I’m an idiot… once registered the license everything works as supposed.

Please, accept my apoligize for wasting your time
Enrico

1 Like