\documentclass{mc}
%%%%% journal  info -- DO NOT CHANGE %%%%%%%%%
\setcounter{page}{219}
\renewcommand\thisnumber{x}
\renewcommand\thisyear {2015}
\renewcommand\thismonth{xxx}
\renewcommand\thisvolume{20}
\renewcommand\datereceived{June 8, 2015}
\renewcommand\dateaccepted{August 22, 2015}
%%%%%%%%   end journal info   %%%%%%%%%%%%%
\usepackage[displaymath,mathlines,pagewise]{lineno}

%Rad zaprimljen dana: 2015-06-08

% Rad prihvacen dana: 22.8.2015.

%%%%% author macros %%%%%%%%%
% place your own macros HERE
%%%%% end %%%%%%%%%

%%%%%%%%%    already loaded packages -- DO NOT RELOAD  %%%%%
% \usepackage{amssymb}
% \usepackage{amsmath,amsthm}
% \usepackage{latexsym}
% \usepackage{amsfonts}
% \usepackage{amsbsyt}
% \usepackage{graphicx}
%%%%%%    end   %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{document}
%\begin{linenumbers}
%%%%% title and author(s):
% \markboth{Author(s)}{Short Title}
% \title{Title}

\markboth{M.\,Kelbert and P.\,Mozgunov}{Shannon's differential entropy asymptotic analysis in a Bayesian problem}
\title[Shannon's differential entropy asymptotic analysis in a Bayesian problem]{Shannon's differential entropy asymptotic analysis in a Bayesian problem}

% multiple authors:
% Please mark \corrauth after the name of the corresponding author.
% different addresses:
\author[M.\,Kelbert, P.\,Mozgunov]{Mark Kelbert\affil{1}\comma\affil{2} and Pavel Mozgunov\affil{1}\comma\affil{3}\comma\corrauth}
\address{\affilnum{1}\ Higher School of Economics, National Research University, 20 Myasnitskaya str., Moscow, 101\,000, RF\\
\affilnum{2}\ Department of Mathematics, Swansea University, Singleton Park, Swansea, SA2 \,8PP,
UK\\\affilnum{3}\ Department of Mathematics and Statistics, Lancaster University, Lancaster, LA1 \,4YF,
UK\\  }
%
%same address:
%\author[AUTHOR1, AUTHOR2 and AUTHOR3]{AUTHOR1, AUTHOR2\corrauth and AUTHOR3}
%\address{address of AUTHOR1, AUTHOR2 and AUTHOR3}
%
\emails{{\tt mark.kelbert@gmail.com}\,\,(M.\,Kelbert), {\tt pmozgunov@gmail.com} (P.\,Mozgunov)}
%

%Faculty of Economics, Moscow Economics National Research University, 20 Myasnitskaya str., Moscow, 101 000, RF


%%%%% Begin Abstract %%%%%%%%%%%
\begin{abstract}
We consider a Bayesian problem of estimating of probability of success in a series of conditionally independent trials with binary outcomes. We study the asymptotic behaviour of the differential entropy for a posterior probability density function conditional on $x$ successes after $n$ conditionally independent trials, when $n \to \infty$. Three particular cases are studied: $x$ is a proportion of $n$; $x$ $\sim n^\beta$, where $0<\beta<1$; either $x$ or $n-x$ is a constant. It is shown that after an appropriate normalization in the first and second case limiting distribution is Gaussian and the differential entropy of a standardized RV converges to the differential entropy of a the Gaussian
distribution. In the third case, the limiting distribution in not Gaussian, but still the asymptotics of the differential entropy can be found explicitly.
\end{abstract}
%%%%% end %%%%%%%%%%%

%%%%% Keywords %%%%%%%%%%%
\keywords{differential entropy, Bayes' formula, Gaussian limit theorem}

%%%% AMS subject classifications %%%%
\ams{94A17, 62B10, 62C10}

%%%% maketitle %%%%%
\maketitle

%%%% Start %%%%%%
\section{Introduction}

Let ${\rm U}$ be a random variable (RV) uniformly distributed in interval $[0,1]$. Given a realization of this RV $p$, consider a sequence of conditionally independent identically distributed $(\xi_i,i=1,2,\ldots)$, where
$\xi_i=1$ with probability $p$ and $\xi_i = 0$ with probability $1-p$. Let $x_i$, each $0$ or $1$, be an outcome in trial $i$. Denote $S_n = \xi_1+ \ldots + \xi_n$ and $x = \sum_{i=1}^{n} x_i$. Note that RVs $(\xi_i)$ are positively correlated. Indeed,
$\mathbb{P}(\xi_i=1,\xi_j=1)\linebreak =\int_0^1p^2 {\rm d}p=1/3$ if $i\not=j$,
but $\mathbb{P}(\xi_i=1)\mathbb{P}(\xi_j=1)=(\int_0^1p {\rm d}p)^2=1/4$.


 The probability that after $n$ trials the exact sequence $(x_i,$ $i =1,...,n) $  will appear equals:
\begin{equation}
 \mathbb{P}(\xi_1 = x_1,...,\xi_n=x_n) = \int_0^1 p^x(1-p)^{n-x}{\rm d} p=\frac{1}{(n+1) {n\choose x}}.
 \end{equation}
This implies that the posterior probability density function (PDF) of the number of $x$ successes after $n$ trials is uniform:
\[ \mathbb{P}(S_n=x) = \frac{1}{(n+1)}, x=0, \ldots, n.\]
It could be easily checked that the sufficient statistics for parameter $p$ is $S_n$. Given the information that after $n$ trials one observes $x$ successes,  the posterior PDF takes the form
\begin{equation}
\displaystyle f_{n}(p|\xi_1 \!=\! x_1,...,\xi_n\!=\!x_n)=f_{n}(p|S_n\!=\!x) = (n+1) {n\choose x} p^x (1-p)^{n-x}, \ 0 \leq p \leq 1.
\label{pdf}
\end{equation}
Note that a conditional distribution given in (\ref{pdf}) is a Beta-distribution $B(x+1,n-x+1)$. \textquotedblleft It is known that a Beta-distribution is asymptotically normal with its mean and variance as $x$ and $(n-x)$ tend to infinity, but this fact is lacking a handy reference\textquotedblright (see \cite[p.1]{mit}). That is why we give the proof of this fact in two cases.

Consider an RV $Z^{(n)}$ on $[0;1]$ with a PDF (\ref{pdf}). Note that $Z^{(n)}$ has the following expectation:
\begin{equation}
\mathbb{E}_x[Z^{(n)}]= \displaystyle \frac{x+1}{n+2}, \end{equation}

\noindent and the following variance:
\begin{equation}\displaystyle \mathbb{V}_x[Z^{(n)}]=\frac{(x+1)(n-x+1)}{(n+3)(n+2)^2}.
\end{equation}
Recall the definition of a differential entropy $h(f)$ of an RV $Z$ with the PDF $f$:
\begin{equation}
h(f)=h_{diff}(f) = - \int_{\mathbb{R}} f(z) {\rm log} (f(z)) {\rm d}z
\end{equation}
with the convention $0 {\rm log} 0 = 0$. When referring to the differential entropy of an RV $Z$ we mean the entropy of its PDF $f$.
Consider a linear transformation $X=b_1Z + b_2$. Then \cite{Cover, Kelbert2014}:
\begin{equation}
h(g)=h(f)+ {\rm log}b_1,
\label{property}
\end{equation}
where  $g$ is a PDF of an RV $X$.
Let $\bar{Z}$ be the standard Gaussian RV with PDF $\varphi$,
\[\varphi(x) = \frac{1}{\sqrt{2 \pi}} e^{-\frac{x^2}{2}}. \]
Then the differential entropy of $\bar{Z}$ equals \cite{Kelbert2014}:
\[h(\varphi) = \frac{1}{2}{\rm log} \left(2 \pi e \right).\]
Recall the definition of the Kullback-Leibler divergence of $g$ from $f$:
\begin{equation}
\mathbb{D}(f||g)= \int_{\mathbb{R}} f(x) {\rm log} \frac{f(x)}{g(x)} {\rm d}x.
\end{equation}

Next, we consider a sequence of distributions $f_n$ of the form (\ref{pdf}) with $x=x(n)$ changing with the size of the sample. The goal of our work is to study the asymptotic behaviour of the differential entropy of the following  RVs:
\begin{enumerate}
\item $Z_{\alpha}^{(n)}$ with PDF $f_{\alpha}^{(n)}$ given in (\ref{pdf}) when $x = x(n) =  \lfloor \alpha n \rfloor$, where $0<\alpha<1$ and $\lfloor a \rfloor$ is the integer part of $a$.
\item $Z_{\beta}^{(n)}$ with PDF $f_{\beta}^{(n)}$ given in (\ref{pdf}) when $x = x(n) = \lfloor n^\beta \rfloor$, where $0<\beta<1$.
\item $Z_{c_1}^{(n)}$ with PDF $f_{c_1}^{(n)}$ given in (\ref{pdf}) when  $x=c_1$ and $Z_{n-c_2}^{(n)}$ with PDF $f_{n-c_2}^{(n)}$ given in (\ref{pdf}) when $n-x(n)=c_2$, where $c_1$ and $c_2$ are some  constants.
\end{enumerate}
Generally, it does not play an important role whether $x(n)$ is integer or not. The analysis also holds for arbitrary positive values. In fact, the main steps of the analysis hold for complex values with positive real part although for complex values of $x$ and $n-x$ the problem loses a direct probabilistic character. We stick to introduced notations in the context of the formulated problem.



\section{Main results}
\begin{theorem}\label{theorem1}
Let $\tilde{Z}_{\alpha}^{(n)}=n^{\frac{1}{2}} (\alpha(1-\alpha))^{-\frac{1}{2}} (Z_{\alpha}^{(n)} - \alpha) $
 be an RV with PDF $\tilde{f}_{\alpha}^{(n)}$. Let $\bar{Z} \sim  \mathcal{N}(0,1)$ be the standard Gaussian RV. Then
\begin{itemize}
\item[(a)] $\tilde{Z}_{\alpha}^{(n)}$ weakly converges to $\bar{Z}$:
\[\tilde{Z}_{\alpha}^{(n)} \Rightarrow \bar{Z} \ {\rm as} \ n \to \infty, \]
\item[{(b)}] the differential entropy of $\tilde{Z}_{\alpha}^{(n)}$converges to the differential entropy of $\bar{Z}$:
\[\lim_{n \rightarrow \infty} h(\tilde{f}_{\alpha}^{(n)})= \frac{1}{2} {\rm log} \left(2 \pi e \right), \]
\item [(c)] the Kullback-Leibler divergence of $\varphi$ from $\tilde{f}_{\alpha}^{(n)}$ tends to $0$ as $n \to \infty$:
\[\lim_{n \rightarrow \infty} \mathbb{D}(\tilde{f}_{\alpha}^{(n)}||\varphi)=0.\]
\end{itemize}
\end{theorem}

\begin{theorem}
\label{theorem2}
Let $\tilde{Z}_{\beta}^{(n)}=n^{1-\beta/2}(Z_{\beta}^{(n)}-n^{\beta-1})$ be an RV with PDF $\tilde{f}_{\beta}^{(n)}$ and $\bar{Z} \sim  \mathcal{N}(0,1)$ then
\begin{itemize}
\item[(a)] $\tilde{Z}_{\beta}^{(n)}$ weakly converges to $\bar{Z}$:
\[\tilde{Z}_{\beta}^{(n)} \Rightarrow \bar{Z} \ {\rm as} \ n \to \infty, \]
\item[{(b)}] the differential entropy of $\tilde{Z}_{\beta}^{(n)}$ converges to the differential entropy of $\bar{Z}$:
 \[ \lim_{n \rightarrow \infty} h(\tilde{f}_{\beta}^{(n)})=\frac{1}{2} {\rm log} \left(2 \pi e \right), \]
\item [(c)] the Kullback-Leibler divergence of $\varphi$ from $\tilde{f}_{\beta}^{(n)}$ tends to $0$ as $n \to \infty$:
\[\lim_{n \rightarrow \infty} \mathbb{D}(\tilde{f}_{\beta}^{(n)}||\varphi)=0.\]
\end{itemize}
\end{theorem}
\begin{theorem}
Let $\tilde{Z}_{c_1}^{(n)}=n{Z}_{c_1}^{(n)}$ be an RV with PDF  $\tilde{f}_{{c_1}}^{(n)}$ and $\tilde{Z}_{n-c_2}^{(n)}=n{Z}_{n-c_2}^{(n)}$ an RV with PDF $\tilde{f}_{n-c_2}^{(n)}$. Let  $H_k =1 +\frac{1}{2} + \ldots + \frac{1}{k}$ denote the partial sum of harmonic series and  $\gamma$ the Euler-Mascheroni constant. Then
\begin{alignat*}{2}
 \mbox{(a)} &\  \lim_{n \rightarrow \infty} h(\tilde{f}_{c_1}^{(n)})  = c_1 + \sum_{i=0}^{c_1-1} {\rm log}(c_1-i)- c_1(H_{c_1} - \gamma)+1,\\
  \mbox{(b)} &\  \lim_{n \rightarrow \infty} h(\tilde{f}_{n-c_2}^{(n)})  = c_2 + \sum_{i=0}^{c_2-1} {\rm log}(c_2-i)- c_2(H_{c_2} - \gamma)+1.
  \end{alignat*}
\label{theorem3}

\end{theorem}

\section{Proof of Theorem 1}

\begin{proof}
{\em{(a)}} Let $x=x(n)=\lfloor \alpha n\rfloor$, where $0<\alpha<1$, and consider a RV \[\tilde{Z}_\alpha^{(n)}=n^{\frac{1}{2}} (\alpha(1-\alpha))^{-\frac{1}{2}} (Z_{\alpha}^{(n)} - \alpha) .\] We proceed by the method of characteristic functions, and establish that
\begin{equation}
\phi(t)=\mathbb{E}[e^{it\tilde{Z}_{\alpha}^{(n)}}]\to e^{-t^2/2}
\label{gaussian}
\end{equation}
for all $t \in \mathbb{R}$.
Indeed,
\begin{alignat*}{2}
 \displaystyle \phi(t) =& \int_0^1e^{it\frac{(p-\alpha)\sqrt{n}}{\sqrt{\alpha(1-\alpha)}}}f^{(n)}_{\alpha}(p){\rm d} p \\
 =& (n+1) {n\choose x}  e^{it\frac{(-\alpha)\sqrt{n}}{\sqrt{\alpha(1-\alpha)}}}\int_0^1e^{it\frac{p\sqrt{n}}{\sqrt{\alpha(1-\alpha)}}}p^x (1-p)^{n-x}{\rm d} p
 \end{alignat*}

\noindent and consider the integral:
\begin{equation}
I(t,\alpha,n)= \int_0^1e^{n(it\frac{p}{\sqrt{\alpha(1-\alpha)n}}+\alpha  {\rm log}p + (1-\alpha) {\rm log}(1-p))}{\rm d} p.
\label{integral11}
\end{equation}
Denote $g(p)=it\frac{p}{\sqrt{\alpha(1-\alpha)n}}+\alpha  {\rm log}p + (1-\alpha) {\rm log}(1-p)$. The integrand in (\ref{integral11}) has a narrow sharp peak, and the integral is
completely dominated by the maximum of ${\rm Re}[g(p)]$ when $n \to \infty$.
For fixed values of $t$, $\alpha$ and $n \to \infty$, it can be studied by the saddle point method \cite[Theorem 1.3, p.170]{Fedoruk1977}:
\begin{equation}
 I(t,\alpha,n) \simeq e^{ng(p^*)}\sqrt{\frac{ 2 \pi}{-n g''(p^*)}} \left(1+ O \left(\frac{1}{n} \right)\right).\end{equation}
Find the point of maximum of ${\rm Re}[g(p)]$ and deform the initial contour $[0,1]$ into the steepest descent contour through the saddle point:
\[p^{*} = \alpha + it \frac{\sqrt{(1-\alpha)\alpha}}{\sqrt{n}} + O \left(\frac{1}{n} \right).\]
So, $\phi(t)$ takes the form:
\[ \phi(t) = e^{- t^2} \displaystyle (n+1) {n\choose x}(p^*)^x (1-p^*)^{n-x} \sqrt{\frac{ 2 \pi}{-n g''(p^*)}} + O \left(\frac{1}{n} \right). \]
Here and below $x= \lfloor \alpha n \rfloor$. Next, by Stirling's formula:
\[(n+1) {n\choose x} \simeq (n+1) \frac{n^n}{x^x (n-x)^{(n-x)}} \sqrt{\frac{n}{2 \pi x (n-x)}}.\]
So, the straightforward computation yields:
 \begin{alignat*}{2}
  \displaystyle (p^*)^x (1-p^*)^{n-x}\simeq &\alpha^x (1-\alpha)^{(n-x)} e^{it \sqrt{(1-\alpha)\alpha n} + \frac{(1-\alpha)t^2}{2} -it \sqrt{(1-\alpha)\alpha n} + \frac{\alpha t^2}{2}}\\
 \displaystyle =& e^{\frac{t^2}{2}} \left(\frac{x}{n} \right) ^x \left(\frac{n-x}{n}\right)^{n-x}. \end{alignat*}
 It can be checked that the next term in the asymptotics of ${\rm log}\,p^*$ (as well as ${\rm log}(1-p^*)$) is decaying to $0$ after multiplication by $\alpha n$ and $(1-\alpha)n$, respectively.

We have for $t \in \mathbb{R}$
\begin{alignat*}{2}
 \displaystyle \phi (t) \simeq  &e^{- t^2} \frac{(n+1)n^n}{x^x (n-x)^{(n-x)}} \sqrt{\frac{n}{2 \pi x (n-x)}}e^{\frac{t^2}{2}} \left(\frac{x}{n} \right) ^x \left(\frac{n-x}{n}\right)^{n-x}\sqrt{\frac{ 2 \pi x (n-x)}{n^3}}\\
  \simeq  &e^{-\frac{t^2}{2}} \end{alignat*}
This fact establishes the pointwise convergence of characteristic function to its Gaussian limit and it completes the proof of part (a).

{\it{(b)}}
Write the differential entropy in the form:
\begin{equation}
\label{hdiff} \displaystyle h(f_{\alpha}^{(n)})= -\left( {\rm log} \left[(n+1) {n\choose x}\right] + (n+1) {n\choose x} x I_1 + (n+1) {n\choose x} (n-x) I_2 \right),\end{equation}
where
\begin{alignat}{2}
I_1 =& \int_0^1  p^x (1-p)^{n-x} {\rm log} \,p \,{\rm d} p,
\label{int1}
\\
I_2 =&\int_0^1  p^x (1-p)^{n-x} {\rm log} (1-p) \,{\rm d} p.
\label{int2}
\end{alignat}
Integrals $I_1$ and $I_2$ can be computed explicitly by reducing to the standard integral
\begin{equation}
\int_0^1x^{\mu-1}(1-x^r)^{\nu-1}{\rm log}\,x\,{\rm d} x = \frac{1}{r^2} B\left(\frac{\mu}{r},\nu \right) \left(\psi\left(\frac{\mu}{r}\right) - \psi\left(\frac{\mu}{r}+\nu\right)\right),
\label{ryz}
\end{equation}
\noindent where $\psi(x)$ is the digamma function, and $B(x,y)$ is the Beta-function \cite[\#4.253.1]{rg2007} and in considering case $r\equiv 1, \mu-1 \equiv x, \nu-1 \equiv n-x$.

For integral $I_1$, we get:
\[ U_1 = (n+1) {n\choose x} x I_1 = -x(\psi(n+2)-\psi(x+1)).\]
Similarly, for the second integral $I_2$, we obtain:
\[U_2 = (n+1) {n\choose x} (n-x) I_2 = -(n-x)(\psi(n+2)-\psi(n-x+1)).\]
After summation of these two integrals and by using the asymptotics for the digamma function \cite[\#8.362.2]{rg2007}, we obtain:
\[U_1+U_2 = x{\rm log}x - n{\rm log}n + (n-x){\rm log}(n-x) - \frac{1}{2}+O \left(\frac{1}{n} \right).  \]
Next, we apply Stirling's formula to the first term in (\ref{hdiff}):
\begin{alignat*}{2}
\displaystyle U_0 =& {\rm log} \left[(n+1) {n\choose x}\right] \\
=& n{\rm log}n -x{\rm log}x -(n-x){\rm log}(n-x)  \\& + \displaystyle  \frac{1}{2}{\rm log}n  - \frac{1}{2}{\rm log}\alpha - \frac{1}{2}{\rm log}(1-\alpha) -{\rm log}(\sqrt{2 \pi}) + O \left(\frac{1}{n} \right). \end{alignat*}
Here, as before, $x= \lfloor \alpha n \rfloor$. So, we obtain the following asymptotics of the differential entropy:
\begin{equation}
 \lim_{n \rightarrow \infty} \left[h(f_{\alpha}^{(n)})-\frac{1}{2} {\rm log}\frac{2 \pi e [\alpha (1-\alpha)]}{n} \right] = 0.
\label{res1}
\end{equation}
Due to (\ref{property}), the differential entropy of RV $\tilde{Z}_\alpha^{(n)}$ has the form:
\begin{equation}
 \lim_{n \rightarrow \infty} \left[h(\tilde{f}_{\alpha}^{(n)}) \right] = \frac{1}{2} {\rm log} \left( 2 \pi e \right).
\end{equation}
{\it{(c)} }By the definition of the the Kullback-Leibler divergence:
\begin{alignat*}{2}
\mathbb{D}(\tilde{f}_{\alpha}^{(n)}||\varphi)= &- h(\tilde{f}_{\alpha}^{(n)}) - \int_0^1 \tilde{f}_{\alpha}^{(n)}(p) \log \varphi(p) {\rm d}p \\
=& - \frac{1}{2} {\rm log} \left( 2 \pi e \right) + \frac{1}{2} \log (2 \pi) + \frac{1}{2} \int_0^1 p^2 \tilde{f}_{\alpha}^{(n)} {\rm d}p + O \left( \frac{1}{n} \right) =  O \left( \frac{1}{n} \right),   \end{alignat*}
 $\int_0^1 p^2 \tilde{f}_{\alpha}^{(n)} {\rm d}p = 1 +O \left( \frac{1}{n} \right) $ is the second moment of $\tilde{Z}_\alpha^{(n)}$. This completes the proof.
 \end{proof}


\section{Proof of Theorem 2}
\begin{proof}
{\it{(a)}} Let $x=x(n)=\lfloor n^\beta\rfloor$, where $0<\beta<1$, and consider $\tilde{Z}_\beta^{(n)}$ such that
\[ \tilde{Z}_{\beta}^{(n)}=n^{1-\beta/2}(Z_{\beta}^{(n)}-n^{\beta-1}).\]
In this case, it is more convenient to proceed by the method of moments. We use the following classical result. Let $f_n$ be a sequence of distribution functions with finite moments $\mu_k(n)$. Let $\mu_k(n)$ tends to $\nu_k$ for each $k$ as $n \to \infty$, where $\nu_k$ are moments of distribution $f$ and the distribution $f$ is uniquely defined by its moments. Then $f_n$ weakly converges to $f$ as $n \to \infty$ \cite{moments}.

Consider RV $ \displaystyle \tilde{Z}_\beta^{(n)} =n^{1-\beta /2}(Z_\beta^{(n)}-n^{\beta -1})$, where $Z_\beta^{(n)}$ has PDF (\ref{pdf}) when $x= \lfloor n^\beta \rfloor$, and compute all moments of $\tilde{Z}_\beta^{(n)}$.
First, $\mathbb{E}(\tilde{Z}_\beta^{(n)}) \to 0$ as $n \to \infty$ because
$\mathbb{E}(Z_\beta^{(n)})= n^{\beta-1}+ O \left(\frac{1}{n} \right)$.
Next, we check that $ \mathbb{E}\left[\left(\tilde{Z}_\beta^{(n)}\right)^2\right]=n^{2(1-\beta/2)} \mathbb{E}(Z_\beta^{(n)}-n^{\beta -1})^2 \to 1$ as $n \to \infty$.
Compute central moments for any $k>1$:
\begin{equation}
\mathbb{E}\left[\left(\tilde{Z}_\beta^{(n)}\right)^k\right] = n^{k-\frac{\beta k}{2}} (1-n^{1-\beta})^{-k} (1-n^{\beta-1})^k \  _2F_1[-k,n^{\beta}+1;n+2;n^{1-\beta}] ,
\label{moments}
\end{equation}
where $_2F_1[-k,n^{\beta}+1;n+2;n^{1-\beta}]$ is the hypergeometric function, which is in this case the polynomial:
\[ _2F_1[-k,n^{\beta}+1;n+2;n^{1-\beta}] =
\sum_{i=0}^{k} (-1)^i {k\choose i} \frac{(n^\beta+1)_i}{(n+2)_i}n^{i(1-\beta)},\] where
$(q)_n$ is the rising Pochhammer symbol. For $n>0$,
\[(q)_n= q(q+1)\ldots (q+n-1)\] \noindent and $(q)_0=1$.


Consider the asymptotics of terms separately:
\[n^{k-\frac{\beta k}{2}} (1-n^{1-\beta})^{-k} (1-n^{\beta-1})^k \simeq O(n^{\frac{k \beta}{2}})\]
\noindent and
\begin{equation} _2F_1[-k,n^{\beta}+1;n+2;n^{1-\beta}] \simeq O(n^{-[0.5+0.5k] \beta}) ,
\label{asymp}
\end{equation}
 where
$\lfloor k \rfloor$ is the integer part of $k$.
For $k$ odd
\begin{equation} n^{k(1-\beta/2)} \mathbb{E}(Z_\beta^{(n)}-n^{\beta -1})^k = O(n^{\frac{k \beta}{2}}) O(n^{-[0.5+0.5k] \beta}) \simeq O(n^{-\beta /2 }) \to 0
\label{odd}
\end{equation} as $n \to \infty$. For $k$ even
\begin{equation} n^{k(1-\beta/2)} \mathbb{E}(Z_\beta^{(n)}-n^{\beta -1})^k = O(n^{\frac{k \beta}{2}}) O(n^{-[0.5+0.5k] \beta}) = O(1).
\label{even}
\end{equation}

We see that every even central moment tends to a constant which is the coefficient in front of term $n^{-[0.5+0.5k] \beta}$ in the hypergeometric function. For $k$ even, we have:
\begin{equation} \displaystyle n^{k(1-\beta/2)} \mathbb{E}(Z_\beta^{(n)}-n^{\beta -1})^k \to (k-1)!.
\label{even}
\end{equation}
These imply that a RV $\tilde{Z}_\beta^{(n)}$ weakly converges to the standard Gaussian RV.


{\it{(b)}}
Write the differential entropy in the form:

\begin{alignat}{2}
 \displaystyle h(f_{\beta}^{(n)})=& -\left( {\rm log}\left[(n+1) {n\choose x} \right] + (n+1) {n\choose x} x I_1 + (n+1) {n\choose x} (n-x) I_2 \right)\nonumber\\ \displaystyle =& -(U_0+U_1 + U_2) ,
\label{hdiff22}
\end{alignat}
where $I_1$ and $I_2$ are defined in (\ref{int1}) and (\ref{int2}) and can be computed explicitly by (\ref{ryz}).

As before, we apply Stirling's formula for $U_0$:
\begin{alignat*}{2}
 U_0 =&   n{\rm log}n -x{\rm log}x -(n-x){\rm log}(n-x) + {\rm log}n \\ &  + \frac{1}{2} (- {\rm log} n^{\beta} - {\rm log}(1-n^{\beta-1})) - \frac{1}{2} {\rm log}( 2 \pi)+ O \left(\frac{1}{n^\beta}  \right).\end{alignat*}
As far as $0<\beta<1$, the remainder tends to $0$ as $n \to \infty$. Note that the rate of decaying depends on parameter $\beta$, contrary to remainder in Theorem 1. Now
$U_1+U_2$ can be computed as follows:
\[\displaystyle U_1+U_2 = x{\rm log}x - n{\rm log}n + (n-x){\rm log}(n-x) - \frac{1}{2} + O \left(\frac{1}{n^\beta}  \right). \]
So, we proved that
 \[\lim_{n \rightarrow \infty} \left[h(f_{\beta}^{(n)})-\frac{1}{2} {\rm log}\frac{2 \pi e (1-n^{\beta-1})}{n^{2-\beta}} \right] = 0 .\]Due to (\ref{property}), the differential entropy of RV $\tilde{Z}_{\beta}^{(n)}$ has the form:
  \[ \lim_{n \rightarrow \infty} h(\tilde{f}_{\beta}^{(n)})=\frac{1}{2} {\rm log} \left(2 \pi e \right). \]
{ \it{(c)}} Similarly, by the definition of the Kullback-Leibler divergence:
\begin{alignat*}{2}
 \mathbb{D}(\tilde{f}_{\beta}^{(n)}||\varphi)=& - h(\tilde{f}_{\beta}^{(n)}) - \int_0^1 \tilde{f}_{\beta}^{(n)}(p) \log \varphi(p) {\rm d}p  \\
=& - \frac{1}{2} {\rm log} \left( 2 \pi e \right) + \frac{1}{2} \log (2 \pi) + \frac{1}{2} \int_0^1 p^2 \tilde{f}_{\beta}^{(n)} {\rm d}p + O \left( \frac{1}{n^{\beta}} \right) =  O \left( \frac{1}{n^{\beta}} \right),   \end{alignat*}
$\int_0^1 p^2 \tilde{f}_{\beta}^{(n)} {\rm d}p = 1 +O \left( \frac{1}{n^{\beta}} \right) $ is the second moment of $\tilde{Z}_\beta^{(n)}$.



\end{proof}



\section{Proof of Theorem 3}
\begin{proof}

\textit{(a)}
Let $x=c_1$, where $c_1$ is some integer constant.
Consider the differential entropy
$ h(f_{c_1}^{(n)})= -(U_0+U_1+U_2)$, where $U_0$, $U_1$ and $U_2$ defined in (\ref{hdiff22}).
Applying Stirling's formula to $U_0$:
\[U_0 = {\rm log}n - {\rm log}(x!) + x{\rm log}n + O \left(\frac{1}{n} \right).\]
Next, we compute $U_1+U_2$ via formula (\ref{ryz}) as before. The only difference will be in the asymptotics of digamma functions \cite[\#8.365.3, \#8.365.4]{rg2007}, because of $x=c_1$, where $c_1$ is constant:

$\psi(n-x+1)\simeq {\rm log}n + \frac{1/2-x}{2n}$, and $\psi(x+1)=H_x - \gamma$, here $H_x$ is the partial sum of harmonic series and $\gamma$ stands for the Euler-Mascheroni constant.
Using that $x=c_1$:
\[ \lim_{n \rightarrow \infty} \left[h(f_{c_1}^{(n)})+{\rm log}n \right] =c_1 + \sum_{i=0}^{c_1-1} {\rm log}(c_1-i)- c_1(H_{c_1} - \gamma)+1.\]
Due to (\ref{property}), it can be written in the following form:
\[   \lim_{n \rightarrow \infty} h(\tilde{f}_{c_1}^{(n)})  = c_1 + \sum_{i=0}^{c_1-1} {\rm log}(c_1-i)- c_1(H_{c_1} - \gamma)+1.\]
\textit{(b)}
Let $n-x(n)=c_2$, where $c_2$ is some integer constant. In a similar way, we compute $h(f_{n-c_2}^{(n)})$, where $n-x=c_2$ and $c_2$ is a constant. The asymptotics of the digamma function is given as follows \cite[\#8.365.4]{rg2007}:
\[\psi(n-x+1)= H_{c_2} - \gamma \ {\rm where} \ x=n-c_2, \]
and the final result for the differential entropy:
\[  h(f_{n-c_2}^{(n)}) =  -{\rm log}n + c_2-c_2(H_{c_2}-\gamma) + \sum_{i=0}^{c_2-1} {\rm log}(c_2-i)+1 + O \left(\frac{1}{n} \right) .\]In terms of standardized RV $\tilde{Z}^{(n)}_{n-c_2}$, due to (\ref{property}) we obtain:
\[  \lim_{n \rightarrow \infty} h(\tilde{f}_{n-c_2}^{(n)})  = c_2 + \sum_{i=0}^{c_2-1} {\rm log}(c_2-i)- c_2(H_{c_2} - \gamma)+1.\]
\end{proof}
\section{Conclusion}
We demonstrated that the limiting distributions of the standardized RV $\tilde{Z}^{(n)}$ when $n \to \infty$ in cases 1 and 2 are Gaussian. However, the asymptotic normality does not imply automatically the limiting form of the differential entropy. In general, the problem of taking the limits under the sign of entropy is rather delicate and was extensively studied in the literature, cf., i.e., \cite{Dobrushin1960,Kelbert2010}. In the third case, the limiting distribution is not Gaussian, but still the asymptotics of the differential entropy can be found explicitly.

For the Bayesian problem studied here, the explicit asymptotic expansions of the Shannon, Renyi, Tsallis and Fisher entropies are presented in \cite{Mozg2015}. The considered problem seems important because of natural extensions of this topic. The so-called weighted differential entropy is the most recent one (see \cite{suhov2014, suhov2015, suhov2015_1} and the reference therein). The weighted version of these entropies is defined in \cite{Mozg2015} and the explicit asymptotic expansions for the stated Bayesian problem are obtained.






\section*{Acknowledgement}

We thank the anonymous referees for useful comments and remarks.

The article was prepared within the framework of a subsidy granted to the HSE by the Government of the Russian Federation for the implementation of the Global Competitiveness Program.

\begin{thebibliography}{99}
  \bibitem{Cover}
 {\sc T.\,M.\,Cover, J.\,M.\,Thomas}, {\em Elements of Information Theory},  John Wiley \& Sons, New York, 2006.

  \bibitem{Dobrushin1960}
 {\sc R.\,L.\,Dobrushin}, {\em Passing to the limit under the sign of the information and entropy}, Theory Prob.Appl. {\bf 5} (1960),  29--37.

       \bibitem{mit}
 {\sc R.\,Dudley}, {\em The Delta-Method and Asymptotics of some Estimators}, Lecture notes on "Topics in Statistics: Nonparametrics and Robustness", MIT, 2005.


 \bibitem{Fedoruk1977}
 {\sc M.\,V.\,Fedoruk}, {\em Saddle Point Method}, Nauka, Moscow, 1977.

    \bibitem{rg2007}
 {\sc I.\,S.\,Gradshteyn, I.\,M.\,Ryzhik,} {\em Table of Integrals, Series, and Product}, Elsevier, New York, 2007.

 \bibitem{Kelbert2010}
 {\sc M.\,Kelbert, Yu.\,Suhov}, {\em Continuity of mutual entropy in the large signal-to-noise ratio limit}, in: {\em Stochastic Analysis}, (D.\,Crisan, Ed.), Springer, Berlin, 2010, 281--299.

 \bibitem{Kelbert2014}
 {\sc M.\,Kelbert, Yu.\,Suhov}, {\em Information Theory and Coding by Example},  Cambridge University Press, Cambridge, 2013.

   \bibitem{Mozg2015}
 {\sc M.\,Kelbert, P.\,Mozgunov}, {\em Asymptotic behaviour of weighted differential entropies in a Bayesian problem}, 2015, arXiv: 1504.01612.

     \bibitem{moments}
 {\sc  Yu.\,V.\,Prokhorov}, {\em  Moments, method of (in probability theory)},  Encyclopedia of Mathematics, 2001, {\tt http://www.encyclopediaofmath}.

       \bibitem{suhov2014}
 {\sc Yu.\,Suhov, S.\,Yasaei Sekeh}, {\em Simple inequalities for weighted entropies}, 2014, arXiv: 1409.4102.

    \bibitem{suhov2015}
 {\sc Yu.\,Suhov, S.\,Yasaei Sekeh, M.\,Kelbert}, {\em Entropy-power inequality for weighted entropy}, 2015, arXiv: 1502.02188.

     \bibitem{suhov2015_1}
 {\sc Yu.\,Suhov, S.\,Yasaei Sekeh, I. Stuhl}, {\em Weighted Gaussian entropy and determinant inequalities}, 2015, arXiv: 1505.01753.



 \end{thebibliography}



%\end{linenumbers}
\end{document}

\textbf{Remark.} Numerical simulation demonstrates that limiting distribution in Theorem 3 is not Gaussian, in particular it has skewness, which can be approximated by skewed Gaussian distribution.




Let $\xi_{\alpha}^{(n)}$ be a RV with posterior PDF $f_{\alpha}^{(n)}$ obtained when $x \sim \alpha n$. We will demonstrate that the limiting distributions when $n \to \infty$ in the cases 1 and 2 are Gaussian. However, the asymptotic normality does not imply automatically the limiting form of differential entropy. In general the problem of taking the limits under the sign of entropy is rather delicate and was extensively studied in literature, cf., i.e., \cite{Dobrushin1960,Kelbert2010}. In the third case the limiting distribution is not Gaussian, but still the asymptotic of differential entropy can be found explicitly.



 We are interested in a posterior distribution after $n$ throws, conditional on event that  $x$ heads appeared. Denote $S_n$ the total number of successes after $n$ throws, say  $S_n= x = \sum_{i=1}^{n}x_i$, $x_i=1$ in the case of head, $0$ in the case of the tail. Here the $\textbf{x}=(x_i,$ $i =1,...,n) $ stands for the realization of throws.



\noindent \textbf{Remark 1.} The first term in (\ref{hdiff1}) is nothing else but the differential entropy of Gaussian RV with variance $\sigma^2 = \frac{\alpha (1-\alpha)}{n}$(cf., i.e., \cite{Kelbert2014}), which agrees with expression for variance:

$$  \mathbb{V}[Z_\alpha^{(n)}|S_n=x] = \frac{(x+1)(n-x+1)}{(n+3)(n+2)^2} \sim \frac{\alpha (1-\alpha)}{n}$$




\begin{equation}
 h(f_{\alpha}^{(n)}) = \frac{1}{2} {\rm log}\frac{2 \pi e [\alpha (1-\alpha)]}{n} + O \left(\frac{1}{n} \right).
 \label{hdiff1}
 \end{equation}


 \textbf{Remark 2.} The first term in (11) is nothing else but the differential entropy of Gaussian RV with variance $\sigma^2 = \frac{1-n^{\beta-1}}{n^{2-\beta}}$, which agrees with expression for variance:
$$ \mathbb{V}[Z_\beta^{(n)}|S_n=x] = \frac{(x+1)(n-x+1)}{(n+3)(n+2)^2} \sim \frac{1-n^{\beta-1}}{n^{2-\beta}}.$$


$$  h(f_{x}^{(n)}) =  -{\rm log}n +x - x(H_x - \gamma) + \sum_{i=1}^{x-1} {\rm log}(x-i) + O \left(\frac{1}{n} \right).$$



AND $\tilde{Z}_{\alpha}^{(n)}=\sqrt{n} \frac{Z_{\alpha}^{(n)} - \alpha}{\sqrt{\alpha(1-\alpha)}} $
