feat(status): realtime visitors (#442)

mydearxym · web-flow · commit f45e9443afb0 · 2021-11-28T21:10:14.000+08:00
* refactor(statistics): online-status &amp; update Telsa stack

* chore(status): online visitors set to 1

* fix(ci): rss tesla client name

* fix(ci): community query arg

* fix(ci): community query arg

* fix(ci): test errors
diff --git a/config/config.exs b/config/config.exs
@@ -139,17 +139,22 @@ config :groupher_server, :cache,
     common: %{
       name: :common,
       size: 5000,
-      minutes: 10
+      seconds: 10 * 60
     },
     user_login: %{
       name: :user_login,
       size: 10_000,
-      minutes: 10_080
+      seconds: 10_080 * 60
     },
     blog_rss: %{
       name: :blog_rss,
       size: 1000,
-      minutes: 15
+      seconds: 15 * 60
+    },
+    online_status: %{
+      name: :online_status,
+      size: 30,
+      seconds: 25
     }
   }
 
@@ -162,9 +167,16 @@ config :groupher_server, Helper.Scheduler,
     # Every 59 minutes
     {"*/59 * * * *", {Helper.Scheduler, :articles_audition, []}},
     # Every 29 minutes
-    {"*/29 * * * *", {Helper.Scheduler, :comments_audition, []}}
+    {"*/29 * * * *", {Helper.Scheduler, :comments_audition, []}},
+    online_status: [
+      # Runs every 20 seconds
+      schedule: {:extended, "*/20"},
+      task: {Helper.Scheduler, :gather_online_status, []}
+    ]
   ]
 
+config :tesla, adapter: Tesla.Adapter.Hackney
+
 # handle background jobs
 config :rihanna,
   jobs_table_name: "background_jobs",
diff --git a/config/prod.exs b/config/prod.exs
@@ -89,7 +89,8 @@ config :groupher_server, :github_oauth,
   client_secret: System.get_env("OAUTH_GITHUB_CLIENT_SECRET"),
   redirect_uri: System.get_env("OAUTH_GITHUB_REDIRECT_URI")
 
-config :groupher_server, :radar_search, ip_service: System.get_env("IP_LOCATE_KEY")
+config :groupher_server, :ip_locate, ip_service: System.get_env("IP_LOCATE_KEY")
+config :groupher_server, :plausible, token: System.get_env("PLAUSIBLE_TOKEN")
 
 config :sentry,
   dsn: System.get_env("SENTRY_DSN"),
diff --git a/lib/groupher_server/accounts/delegates/profile.ex b/lib/groupher_server/accounts/delegates/profile.ex
@@ -13,7 +13,7 @@ defmodule GroupherServer.Accounts.Delegate.Profile do
 
   alias GroupherServer.Accounts.Delegate.Fans
 
-  alias Helper.{Guardian, ORM, QueryBuilder, RadarSearch}
+  alias Helper.{Guardian, ORM, QueryBuilder, IP2City}
   alias Ecto.Multi
 
   @default_user_meta Embeds.UserMeta.default_meta()
@@ -92,7 +92,7 @@ defmodule GroupherServer.Accounts.Delegate.Profile do
   update geo info for user, include geo_city & remote ip
   """
   def update_geo(%User{geo_city: geo_city} = user, remote_ip) when is_nil(geo_city) do
-    case RadarSearch.locate_city(remote_ip) do
+    case IP2City.locate_city(remote_ip) do
       {:ok, city} ->
         update_profile(user, %{geo_city: city, remote_ip: remote_ip})
 
diff --git a/lib/groupher_server/application.ex b/lib/groupher_server/application.ex
@@ -47,6 +47,7 @@ defmodule GroupherServer.Application do
     # worker(Cachex, [:common, Cache.config(:common)], id: :common),
     # worker(Cachex, [:user_login, Cache.config(:user_login)], id: :user_login),
     # worker(Cachex, [:blog_rss, Cache.config(:blog_rss)], id: :blog_rss),
+
     @cache_pool
     |> Map.keys()
     |> Enum.reduce([], fn key, acc ->
diff --git a/lib/groupher_server/cms/delegates/blog_curd.ex b/lib/groupher_server/cms/delegates/blog_curd.ex
@@ -98,8 +98,7 @@ defmodule GroupherServer.CMS.Delegate.BlogCURD do
   end
 
   defp get_rssinfo_and_cache(rss) do
-    # {:ok, feed} = RSS.get(rss)
-    with {:ok, rssinfo} <- RSS.get(rss) do
+    with {:ok, rssinfo} <- RSS.query(rss) do
       Cache.put(@cache_pool, rss, rssinfo)
       {:ok, rssinfo}
     else
diff --git a/lib/groupher_server/cms/delegates/community_operation.ex b/lib/groupher_server/cms/delegates/community_operation.ex
@@ -4,7 +4,7 @@ defmodule GroupherServer.CMS.Delegate.CommunityOperation do
   """
   import ShortMaps
 
-  alias Helper.{Certification, RadarSearch, ORM}
+  alias Helper.{Certification, IP2City, ORM}
 
   alias GroupherServer.{Accounts, CMS, Repo}
 
@@ -285,7 +285,7 @@ defmodule GroupherServer.CMS.Delegate.CommunityOperation do
   end
 
   defp get_user_geocity(nil, remote_ip) do
-    case RadarSearch.locate_city(remote_ip) do
+    case IP2City.locate_city(remote_ip) do
       {:ok, city} -> {:ok, city}
       {:error, _} -> {:error, "update_community geo error"}
     end
diff --git a/lib/groupher_server/statistics/delegates/status.ex b/lib/groupher_server/statistics/delegates/status.ex
@@ -7,23 +7,54 @@ defmodule GroupherServer.Statistics.Delegate.Status do
   import ShortMaps
 
   alias GroupherServer.CMS
-  alias CMS.Model.{Post, Job, Repo, Community, Thread, Category, ArticleTag}
-  alias Helper.ORM
 
+  alias CMS.Model.{
+    Post,
+    Job,
+    Guide,
+    Meetup,
+    Drink,
+    Blog,
+    Radar,
+    Works,
+    Drink,
+    Community,
+    Thread,
+    Category,
+    ArticleTag
+  }
+
+  alias Helper.{ORM, Cache}
+
+  @cache_pool :online_status
   @count_filter %{page: 1, size: 1}
 
+  def online_status() do
+    with {:ok, realtime_visitors} <- Cache.get(@cache_pool, :realtime_visitors) do
+      {:ok, %{realtime_visitors: realtime_visitors}}
+    else
+      _ ->
+        {:ok, %{realtime_visitors: 1}}
+    end
+  end
+
   def count_status do
     {:ok, %{total_count: communities_count}} = find_total_count(Community)
     {:ok, %{total_count: posts_count}} = find_total_count(Post)
     {:ok, %{total_count: jobs_count}} = find_total_count(Job)
-    {:ok, %{total_count: repos_count}} = find_total_count(Repo)
+    {:ok, %{total_count: blogs_count}} = find_total_count(Blog)
+    {:ok, %{total_count: works_count}} = find_total_count(Works)
+    {:ok, %{total_count: meetups_count}} = find_total_count(Meetup)
+    {:ok, %{total_count: guides_count}} = find_total_count(Guide)
+    {:ok, %{total_count: radars_count}} = find_total_count(Radar)
+    {:ok, %{total_count: drinks_count}} = find_total_count(Drink)
 
     {:ok, %{total_count: threads_count}} = find_total_count(Thread)
     {:ok, %{total_count: article_tags_count}} = find_total_count(ArticleTag)
     {:ok, %{total_count: categories_count}} = find_total_count(Category)
 
     {:ok,
-     ~m(communities_count posts_count jobs_count repos_count threads_count article_tags_count categories_count)a}
+     ~m(communities_count posts_count jobs_count works_count meetups_count guides_count radars_count blogs_count drinks_count threads_count article_tags_count categories_count)a}
   end
 
   defp find_total_count(queryable), do: ORM.find_all(queryable, @count_filter)
diff --git a/lib/groupher_server/statistics/statistics.ex b/lib/groupher_server/statistics/statistics.ex
@@ -21,4 +21,5 @@ defmodule GroupherServer.Statistics do
 
   # countStatus
   defdelegate count_status(), to: Status
+  defdelegate online_status(), to: Status
 end
diff --git a/lib/groupher_server_web/resolvers/statistics_resolver.ex b/lib/groupher_server_web/resolvers/statistics_resolver.ex
@@ -20,6 +20,10 @@ defmodule GroupherServerWeb.Resolvers.Statistics do
     Statistics.list_cities_info()
   end
 
+  def online_status(_root, _args, _info) do
+    Statistics.online_status()
+  end
+
   def count_status(_root, _args, _info) do
     Statistics.count_status()
   end
diff --git a/lib/groupher_server_web/schema/statistics/statistics_queries.ex b/lib/groupher_server_web/schema/statistics/statistics_queries.ex
@@ -10,6 +10,13 @@ defmodule GroupherServerWeb.Schema.Statistics.Queries do
       resolve(&R.Statistics.list_cities_geo_info/3)
     end
 
+    @desc "basic online status"
+    field :online_status, :online_status_info do
+      arg(:freshkey, :string)
+
+      resolve(&R.Statistics.online_status/3)
+    end
+
     @desc "basic site info in total counts format"
     field :count_status, :count_status_info do
       middleware(M.Passport, claim: "cms->root")
diff --git a/lib/groupher_server_web/schema/statistics/statistics_types.ex b/lib/groupher_server_web/schema/statistics/statistics_types.ex
@@ -11,11 +11,20 @@ defmodule GroupherServerWeb.Schema.Statistics.Types do
     field(:date, :date)
   end
 
+  object :online_status_info do
+    field(:realtime_visitors, :integer)
+  end
+
   object :count_status_info do
     field(:communities_count, :integer)
     field(:posts_count, :integer)
     field(:jobs_count, :integer)
-    field(:repos_count, :integer)
+    field(:works_count, :integer)
+    field(:meetups_count, :integer)
+    field(:radars_count, :integer)
+    field(:blogs_count, :integer)
+    field(:guides_count, :integer)
+    field(:drinks_count, :integer)
 
     field(:categories_count, :integer)
     field(:article_tags_count, :integer)
diff --git a/lib/helper/RSS.ex b/lib/helper/RSS.ex
@@ -2,10 +2,17 @@ defmodule Helper.RSS do
   @moduledoc """
   RSS get and parser
   """
+  use Tesla, only: [:get]
   import Helper.Utils, only: [done: 1]
 
-  def get(rss) do
-    with {:ok, %{body: body}} <- HTTPoison.get(rss),
+  @timeout_limit 4000
+
+  plug(Tesla.Middleware.Retry, delay: 200, max_retries: 2)
+  plug(Tesla.Middleware.Timeout, timeout: @timeout_limit)
+  plug(Tesla.Middleware.JSON)
+
+  def query(rss) do
+    with {:ok, %{body: body}} <- get(rss),
          {:ok, blog_rss} <- rss_parser(body) do
       blog_rss |> Map.merge(%{rss: rss}) |> done
     else
diff --git a/lib/helper/cache.ex b/lib/helper/cache.ex
@@ -10,7 +10,7 @@ defmodule Helper.Cache do
   def config(pool_name) do
     [
       limit: limit(size: @cache_pool[pool_name].size, policy: Cachex.Policy.LRW, reclaim: 0.1),
-      expiration: expiration(default: :timer.minutes(@cache_pool[pool_name].minutes))
+      expiration: expiration(default: :timer.seconds(@cache_pool[pool_name].seconds))
     ]
   end
 
diff --git a/lib/helper/ip_2_city.ex b/lib/helper/ip_2_city.ex
@@ -1,4 +1,4 @@
-defmodule Helper.RadarSearch do
+defmodule Helper.IP2City do
   @moduledoc """
   find city info by ip
   refer: https://lbs.amap.com/api/webservice/guide/api/ipconfig/?sug_index=0
@@ -8,6 +8,7 @@ defmodule Helper.RadarSearch do
 
   @endpoint "https://restapi.amap.com/v3/ip"
   @timeout_limit 5000
+  @service_key get_config(:ip_locate, :ip_service)
 
   # plug(Tesla.Middleware.BaseUrl, "https://restapi.amap.com/v3/ip")
   plug(Tesla.Middleware.Retry, delay: 200, max_retries: 2)
@@ -30,7 +31,7 @@ defmodule Helper.RadarSearch do
 
   # http://ip.yqie.com/search.aspx?searchword=%E6%88%90%E9%83%BD%E5%B8%82
   def locate_city(ip) do
-    query = [ip: ip, key: get_config(:radar_search, :ip_service)]
+    query = [ip: ip, key: @service_key]
 
     with true <- Mix.env() !== :test do
       case get(@endpoint, query: query) do
diff --git a/lib/helper/oauth2/github.ex b/lib/helper/oauth2/github.ex
@@ -12,11 +12,12 @@ defmodule Helper.OAuth2.Github do
   plug(Tesla.Middleware.BaseUrl, "https://github.com/login/oauth")
   # plug(Tesla.Middleware.BaseUrl, "https://www.github.com/login/oauth")
   # plug(Tesla.Middleware.BaseUrl, "https://api.github.com/login/oauth")
-  plug(Tesla.Middleware.Headers, %{
-    "User-Agent" => "groupher server"
-    # "Accept" => "application/json"
-    # "Accept" => "application/json;application/vnd.github.jean-grey-preview+json"
-  })
+  plug(Tesla.Middleware.Headers, [{"User-Agent", "groupher server"}])
+  # plug(Tesla.Middleware.Headers, %{
+  # "User-Agent" => "groupher server"
+  # "Accept" => "application/json"
+  # "Accept" => "application/json;application/vnd.github.jean-grey-preview+json"
+  # })
 
   plug(Tesla.Middleware.Retry, delay: 200, max_retries: 2)
   plug(Tesla.Middleware.Timeout, timeout: @timeout_limit)
diff --git a/lib/helper/plausible.ex b/lib/helper/plausible.ex
@@ -0,0 +1,48 @@
+defmodule Helper.Plausible do
+  @moduledoc """
+  find city info by ip
+  refer: https://lbs.amap.com/api/webservice/guide/api/ipconfig/?sug_index=0
+  """
+  use Tesla, only: [:get]
+  import Helper.Utils, only: [get_config: 2]
+
+  alias Helper.Cache
+
+  @endpoint "https://plausible.io"
+  @realtime_visitors_query "/api/v1/stats/realtime/visitors"
+  @timeout_limit 4000
+
+  @site_id "coderplanets.com"
+  @service_key get_config(:plausible, :token)
+
+  @cache_pool :online_status
+
+  plug(Tesla.Middleware.BaseUrl, @endpoint)
+  plug(Tesla.Middleware.Headers, [{"Authorization", "Bearer #{@service_key}"}])
+  plug(Tesla.Middleware.Retry, delay: 200, max_retries: 2)
+  plug(Tesla.Middleware.Timeout, timeout: @timeout_limit)
+  plug(Tesla.Middleware.JSON)
+
+  def realtime_visitors() do
+    query = [site_id: @site_id]
+    path = "#{@realtime_visitors_query}"
+    # NOTICE: DO NOT use Tesla.get, otherwise the middleware will not woking
+    # see https://github.com/teamon/tesla/issues/88
+    # with true <- Mix.env() !== :test do
+    with {:ok, %{body: body}} <- get(path, query: query) do
+      case is_number(body) do
+        true ->
+          Cache.put(@cache_pool, :realtime_visitors, body)
+          {:ok, body}
+
+        false ->
+          {:ok, 1}
+      end
+    else
+      error ->
+        IO.inspect(error, label: "got error")
+        Cache.put(@cache_pool, :realtime_visitors, 1)
+        {:ok, 1}
+    end
+  end
+end
diff --git a/lib/helper/scheduler.ex b/lib/helper/scheduler.ex
@@ -7,6 +7,7 @@ defmodule Helper.Scheduler do
   import Helper.Utils, only: [get_config: 2, done: 1]
   alias GroupherServer.CMS
   alias CMS.Delegate.Hooks
+  alias Helper.Plausible
 
   @article_threads get_config(:article, :threads)
 
@@ -58,4 +59,10 @@ defmodule Helper.Scheduler do
       |> done
     end
   end
+
+  def gather_online_status() do
+    with true <- Mix.env() !== :test do
+      Plausible.realtime_visitors()
+    end
+  end
 end
diff --git a/mix.exs b/mix.exs
@@ -71,7 +71,9 @@ defmodule GroupherServer.Mixfile do
       {:comeonin, "~> 5.3.2"},
       # CORS
       {:corsica, "~> 1.1.2"},
-      {:tesla, "~> 0.10.0"},
+      {:tesla, "~> 1.4"},
+      # optional, but recommended adapter
+      {:hackney, "~> 1.17.4"},
       # only used for tesla's JSON-encoder
       {:poison, "~> 3.1"},
       # for fake data in test env
diff --git a/mix.lock b/mix.lock
@@ -96,7 +96,7 @@
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
   "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm", "94884f84783fc1ba027aba8fe8a7dae4aad78c98e9f9c76667ec3471585c08c6"},
   "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"},
-  "tesla": {:hex, :tesla, "0.10.0", "e588c7e7f1c0866c81eeed5c38f02a4a94d6309eede336c1e6ca08b0a95abd3f", [:mix], [{:exjsx, ">= 0.1.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm", "bb10f37d124e2a04950e1642437fc30368cd992bd7370b38eeb3af1ebf1c2029"},
+  "tesla": {:hex, :tesla, "1.4.3", "f5a494e08fb1abe4fd9c28abb17f3d9b62b8f6fc492860baa91efb1aab61c8a0", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "e0755bb664bf4d664af72931f320c97adbf89da4586670f4864bf259b5750386"},
   "timex": {:hex, :timex, "3.7.5", "3eca56e23bfa4e0848f0b0a29a92fa20af251a975116c6d504966e8a90516dfd", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "a15608dca680f2ef663d71c95842c67f0af08a0f3b1d00e17bbd22872e2874e4"},
   "tzdata": {:hex, :tzdata, "1.1.0", "72f5babaa9390d0f131465c8702fa76da0919e37ba32baa90d93c583301a8359", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "18f453739b48d3dc5bcf0e8906d2dc112bb40baafe2c707596d89f3c8dd14034"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
diff --git a/test/groupher_server/cms/comments/works_comment_replies_test.exs b/test/groupher_server/cms/comments/works_comment_replies_test.exs
@@ -144,6 +144,7 @@ defmodule GroupherServer.Test.CMS.Comments.WorksCommentReplies do
   end
 
   describe "[paged article comment replies]" do
+    @tag :wip
     test "can get paged replies of a parent comment", ~m(works user)a do
       {:ok, parent_comment} = CMS.create_comment(:works, works.id, mock_comment(), user)
       {:ok, paged_replies} = CMS.paged_comment_replies(parent_comment.id, %{page: 1, size: 20})
diff --git a/test/groupher_server/cms/community/community_meta_test.exs b/test/groupher_server/cms/community/community_meta_test.exs
@@ -29,7 +29,6 @@ defmodule GroupherServer.Test.Community.CommunityMeta do
       assert community.meta |> strip_struct == @default_meta
     end
 
-    @tag :wip
     test "update legacy community should add default meta", ~m(community)a do
       assert is_nil(community.meta)
 
diff --git a/test/groupher_server/cms/community/community_test.exs b/test/groupher_server/cms/community/community_test.exs
diff --git a/test/groupher_server/statistics/statistics_test.exs b/test/groupher_server/statistics/statistics_test.exs
diff --git a/test/groupher_server/statistics/status_test.exs b/test/groupher_server/statistics/status_test.exs
diff --git a/test/groupher_server_web/mutation/cms/cms_test.exs b/test/groupher_server_web/mutation/cms/cms_test.exs
diff --git a/test/groupher_server_web/query/cms/cms_test.exs b/test/groupher_server_web/query/cms/cms_test.exs
diff --git a/test/groupher_server_web/query/cms/community_meta_test.exs b/test/groupher_server_web/query/cms/community_meta_test.exs
diff --git a/test/groupher_server_web/query/statistics/statistics_test.exs b/test/groupher_server_web/query/statistics/statistics_test.exs
diff --git a/test/support/factory.ex b/test/support/factory.ex
